[llvm] AMDGPU: Directly handle all atomicrmw cases in SIISelLowering (PR #102439)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 8 01:55:30 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Matt Arsenault (arsenm)
<details>
<summary>Changes</summary>
We were deferring to the base AMDGPUISelLowering implementation
in some cases, which was harder to follow. This simplifies additional
new handling in future patches.
---
Full diff: https://github.com/llvm/llvm-project/pull/102439.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (-28)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h (-2)
- (modified) llvm/lib/Target/AMDGPU/R600ISelLowering.cpp (+21-2)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+64-5)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 862b5c7e3e3d7..3fc02c6da37cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5999,34 +5999,6 @@ bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks
}
-TargetLowering::AtomicExpansionKind
-AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
- switch (RMW->getOperation()) {
- case AtomicRMWInst::Nand:
- case AtomicRMWInst::FAdd:
- case AtomicRMWInst::FSub:
- case AtomicRMWInst::FMax:
- case AtomicRMWInst::FMin:
- return AtomicExpansionKind::CmpXChg;
- case AtomicRMWInst::Xchg: {
- const DataLayout &DL = RMW->getFunction()->getDataLayout();
- unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
- if (ValSize == 32 || ValSize == 64)
- return AtomicExpansionKind::None;
- return AtomicExpansionKind::CmpXChg;
- }
- default: {
- if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
- unsigned Size = IntTy->getBitWidth();
- if (Size == 32 || Size == 64)
- return AtomicExpansionKind::None;
- }
-
- return AtomicExpansionKind::CmpXChg;
- }
- }
-}
-
/// Whether it is profitable to sink the operands of an
/// Instruction I to the basic block of I.
/// This helps using several modifiers (like abs and neg) more often.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 37572af3897f2..1a5244f7ec809 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -388,8 +388,6 @@ class AMDGPUTargetLowering : public TargetLowering {
return MVT::i32;
}
- AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
-
bool shouldSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
};
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index 7e0d96622f3c5..c79688bf038be 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -2175,14 +2175,33 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
TargetLowering::AtomicExpansionKind
R600TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
switch (RMW->getOperation()) {
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMax:
+ case AtomicRMWInst::FMin:
+ return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
// FIXME: Cayman at least appears to have instructions for this, but the
// instruction defintions appear to be missing.
return AtomicExpansionKind::CmpXChg;
+ case AtomicRMWInst::Xchg: {
+ const DataLayout &DL = RMW->getFunction()->getDataLayout();
+ unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
+ if (ValSize == 32 || ValSize == 64)
+ return AtomicExpansionKind::None;
+ return AtomicExpansionKind::CmpXChg;
+ }
default:
- break;
+ if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
+ unsigned Size = IntTy->getBitWidth();
+ if (Size == 32 || Size == 64)
+ return AtomicExpansionKind::None;
+ }
+
+ return AtomicExpansionKind::CmpXChg;
}
- return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ llvm_unreachable("covered atomicrmw op switch");
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4e9c271197613..8497aa33449e9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16114,6 +16114,39 @@ static bool isBFloat2(Type *Ty) {
return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
}
+/// \return true if atomicrmw integer ops work for the type.
+static bool isAtomicRMWLegalIntTy(Type *Ty) {
+ if (auto *IT = dyn_cast<IntegerType>(Ty)) {
+ unsigned BW = IT->getBitWidth();
+ return BW == 32 || BW == 64;
+ }
+
+ return false;
+}
+
+/// \return true if this atomicrmw xchg type can be selected.
+static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
+ Type *Ty = RMW->getType();
+ if (isAtomicRMWLegalIntTy(Ty))
+ return true;
+
+ if (PointerType *PT = dyn_cast<PointerType>(Ty)) {
+ const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
+ unsigned BW = DL.getPointerSizeInBits(PT->getAddressSpace());
+ return BW == 32 || BW == 64;
+ }
+
+ if (Ty->isFloatTy() || Ty->isDoubleTy())
+ return true;
+
+ if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty)) {
+ return VT->getNumElements() == 2 &&
+ VT->getElementType()->getPrimitiveSizeInBits() == 16;
+ }
+
+ return false;
+}
+
/// \returns true if it's valid to emit a native instruction for \p RMW, based
/// on the properties of the target memory.
static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
@@ -16142,6 +16175,14 @@ static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
.getValueAsBool();
}
+/// \return Action to perform on AtomicRMWInsts for integer operations.
+static TargetLowering::AtomicExpansionKind
+atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
+ return isAtomicRMWLegalIntTy(RMW->getType())
+ ? TargetLowering::AtomicExpansionKind::None
+ : TargetLowering::AtomicExpansionKind::CmpXChg;
+}
+
TargetLowering::AtomicExpansionKind
SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
unsigned AS = RMW->getPointerAddressSpace();
@@ -16161,7 +16202,22 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
SSID == SyncScope::System ||
SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
- switch (RMW->getOperation()) {
+ auto Op = RMW->getOperation();
+ switch (Op) {
+ case AtomicRMWInst::Xchg: {
+ // PCIe supports add and xchg for system atomics.
+ return isAtomicRMWLegalXChgTy(RMW)
+ ? TargetLowering::AtomicExpansionKind::None
+ : TargetLowering::AtomicExpansionKind::CmpXChg;
+
+ // PCIe supports add and xchg for system atomics.
+ return atomicSupportedIfLegalIntType(RMW);
+ }
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::And:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ return atomicSupportedIfLegalIntType(RMW);
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor: {
@@ -16173,7 +16229,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
return AtomicExpansionKind::Expand;
}
- break;
+ return atomicSupportedIfLegalIntType(RMW);
}
case AtomicRMWInst::FAdd: {
Type *Ty = RMW->getType();
@@ -16335,13 +16391,16 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
if (HasSystemScope)
return AtomicExpansionKind::CmpXChg;
}
- break;
+
+ return atomicSupportedIfLegalIntType(RMW);
}
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FSub:
default:
- break;
+ return AtomicExpansionKind::CmpXChg;
}
- return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ llvm_unreachable("covered atomicrmw op switch");
}
TargetLowering::AtomicExpansionKind
``````````
</details>
https://github.com/llvm/llvm-project/pull/102439
More information about the llvm-commits
mailing list