[llvm] AMDGPU: Don't bitcast float typed atomic store in IR (PR #90116)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 26 00:56:10 PDT 2024
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/90116
>From 9a094b76e567ac666704f8b3447be7286f8c29a0 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 25 Apr 2024 19:10:11 +0200
Subject: [PATCH 1/2] DAG: Simplify demanded bits for truncating atomic_store
It's really unfortunate that STORE and ATOMIC_STORE are separate
opcodes. This duplicates a basic simplify demanded for the truncating
case. This avoids some AMDGPU lit regressions in a future patch.
I'm not sure how to craft a test that exposes this without first
introducing the regressions by promoting half to i16.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 20 +++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aa746f1c7b7b3b..f115a39a6953ce 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -530,6 +530,7 @@ namespace {
bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
SDValue visitSTORE(SDNode *N);
+ SDValue visitATOMIC_STORE(SDNode *N);
SDValue visitLIFETIME_END(SDNode *N);
SDValue visitINSERT_VECTOR_ELT(SDNode *N);
SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
@@ -1909,6 +1910,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::BR_CC: return visitBR_CC(N);
case ISD::LOAD: return visitLOAD(N);
case ISD::STORE: return visitSTORE(N);
+ case ISD::ATOMIC_STORE: return visitATOMIC_STORE(N);
case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
@@ -21096,6 +21098,24 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
ST->getMemOperand()->getFlags());
}
+SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ EVT VT = Val.getValueType();
+ EVT MemVT = ST->getMemoryVT();
+
+ if (MemVT.bitsLT(VT)) { // Is truncating store
+ APInt TruncDemandedBits = APInt::getLowBitsSet(VT.getScalarSizeInBits(),
+ MemVT.getScalarSizeInBits());
+ // See if we can simplify the operation with SimplifyDemandedBits, which
+ // only works if the value has a single use.
+ if (SimplifyDemandedBits(Val, TruncDemandedBits))
+ return SDValue(N, 0);
+ }
+
+ return SDValue();
+}
+
SDValue DAGCombiner::visitSTORE(SDNode *N) {
StoreSDNode *ST = cast<StoreSDNode>(N);
SDValue Chain = ST->getChain();
>From abe66f71bdf73eb92022166c23ad8fcb591af9db Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 25 Apr 2024 16:19:22 +0200
Subject: [PATCH 2/2] AMDGPU: Don't bitcast float typed atomic store in IR
Implement the promotion in the DAG.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 29 +++++++++++-----
.../SelectionDAG/LegalizeFloatTypes.cpp | 34 +++++++++++++++++++
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 2 ++
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 12 +++++++
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 5 +++
5 files changed, 73 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 8413cd60135f7f..40a948bcf623de 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4946,7 +4946,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
OVT = Node->getOperand(0).getSimpleValueType();
}
- if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
+ if (Node->getOpcode() == ISD::ATOMIC_STORE ||
+ Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
Node->getOpcode() == ISD::STRICT_FSETCC ||
Node->getOpcode() == ISD::STRICT_FSETCCS)
@@ -5557,7 +5558,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Results.push_back(CvtVec);
break;
}
- case ISD::ATOMIC_SWAP: {
+ case ISD::ATOMIC_SWAP:
+ case ISD::ATOMIC_STORE: {
AtomicSDNode *AM = cast<AtomicSDNode>(Node);
SDLoc SL(Node);
SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
@@ -5566,13 +5568,22 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
"unexpected atomic_swap with illegal type");
- SDValue NewAtomic
- = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
- DAG.getVTList(NVT, MVT::Other),
- { AM->getChain(), AM->getBasePtr(), CastVal },
- AM->getMemOperand());
- Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
- Results.push_back(NewAtomic.getValue(1));
+ SDValue Op0 = AM->getBasePtr();
+ SDValue Op1 = CastVal;
+
+ // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
+ // but really it should merge with ISD::STORE.
+ if (AM->getOpcode() == ISD::ATOMIC_STORE)
+ std::swap(Op0, Op1);
+
+ SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(),
+ Op0, Op1, AM->getMemOperand());
+
+ if (AM->getOpcode() != ISD::ATOMIC_STORE) {
+ Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
+ Results.push_back(NewAtomic.getValue(1));
+ } else
+ Results.push_back(NewAtomic);
break;
}
case ISD::SPLAT_VECTOR: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 7685bc73cf9652..5f21f65cab6273 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2249,6 +2249,7 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
+ case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
}
// clang-format on
@@ -2371,6 +2372,23 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
ST->getMemOperand());
}
+SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
+ unsigned OpNo) {
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ SDLoc DL(N);
+
+ SDValue Promoted = GetPromotedFloat(Val);
+ EVT VT = ST->getOperand(1).getValueType();
+ EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+
+ SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
+ DL, IVT, Promoted);
+
+ return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
+ ST->getBasePtr(), ST->getMemOperand());
+}
+
//===----------------------------------------------------------------------===//
// Float Result Promotion
//===----------------------------------------------------------------------===//
@@ -3154,6 +3172,9 @@ bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
+ case ISD::ATOMIC_STORE:
+ Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
+ break;
case ISD::STACKMAP:
Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
break;
@@ -3307,6 +3328,19 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
ST->getMemOperand());
}
+SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "Can only soften the stored value!");
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ SDLoc dl(N);
+
+ SDValue Promoted = GetSoftPromotedHalf(Val);
+ return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
+ ST->getChain(), Promoted, ST->getBasePtr(),
+ ST->getMemOperand());
+}
+
SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 9c855e55855312..fad0067564bf4d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -708,6 +708,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
+ SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
@@ -751,6 +752,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
+ SDValue SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f4a747784d1fd2..ec2e0f3cbbf3c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -148,6 +148,18 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::i128, Promote);
AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f64, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 72661a8d29f816..d00888544ea799 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -230,6 +230,11 @@ class AMDGPUTargetLowering : public TargetLowering {
bool isCheapToSpeculateCtlz(Type *Ty) const override;
bool isSDNodeAlwaysUniform(const SDNode *N) const override;
+
+ AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override {
+ return AtomicExpansionKind::None;
+ }
+
static CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg);
static CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg);
More information about the llvm-commits
mailing list