[llvm] AMDGPU: Directly handle all atomicrmw cases in SIISelLowering (PR #102439)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 8 01:53:37 PDT 2024
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/102439
We were deferring to the base AMDGPUISelLowering implementation
in some cases, which was harder to follow. This simplifies additional
new handling in future patches.
>From 2fb750529a2c45089885350d70c0c58b00da689a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 8 Aug 2024 12:33:58 +0400
Subject: [PATCH] AMDGPU: Directly handle all atomicrmw cases in SIISelLowering
We were deferring to the base AMDGPUISelLowering implementation
in some cases, which was harder to follow. This simplifies additional
new handling in future patches.
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 28 --------
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 2 -
llvm/lib/Target/AMDGPU/R600ISelLowering.cpp | 23 ++++++-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 69 +++++++++++++++++--
4 files changed, 85 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 862b5c7e3e3d7..3fc02c6da37cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5999,34 +5999,6 @@ bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks
}
-TargetLowering::AtomicExpansionKind
-AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
- switch (RMW->getOperation()) {
- case AtomicRMWInst::Nand:
- case AtomicRMWInst::FAdd:
- case AtomicRMWInst::FSub:
- case AtomicRMWInst::FMax:
- case AtomicRMWInst::FMin:
- return AtomicExpansionKind::CmpXChg;
- case AtomicRMWInst::Xchg: {
- const DataLayout &DL = RMW->getFunction()->getDataLayout();
- unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
- if (ValSize == 32 || ValSize == 64)
- return AtomicExpansionKind::None;
- return AtomicExpansionKind::CmpXChg;
- }
- default: {
- if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
- unsigned Size = IntTy->getBitWidth();
- if (Size == 32 || Size == 64)
- return AtomicExpansionKind::None;
- }
-
- return AtomicExpansionKind::CmpXChg;
- }
- }
-}
-
/// Whether it is profitable to sink the operands of an
/// Instruction I to the basic block of I.
/// This helps using several modifiers (like abs and neg) more often.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 37572af3897f2..1a5244f7ec809 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -388,8 +388,6 @@ class AMDGPUTargetLowering : public TargetLowering {
return MVT::i32;
}
- AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
-
bool shouldSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
};
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index 7e0d96622f3c5..c79688bf038be 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -2175,14 +2175,33 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
TargetLowering::AtomicExpansionKind
R600TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
switch (RMW->getOperation()) {
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ case AtomicRMWInst::FMax:
+ case AtomicRMWInst::FMin:
+ return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
// FIXME: Cayman at least appears to have instructions for this, but the
// instruction defintions appear to be missing.
return AtomicExpansionKind::CmpXChg;
+ case AtomicRMWInst::Xchg: {
+ const DataLayout &DL = RMW->getFunction()->getDataLayout();
+ unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
+ if (ValSize == 32 || ValSize == 64)
+ return AtomicExpansionKind::None;
+ return AtomicExpansionKind::CmpXChg;
+ }
default:
- break;
+ if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
+ unsigned Size = IntTy->getBitWidth();
+ if (Size == 32 || Size == 64)
+ return AtomicExpansionKind::None;
+ }
+
+ return AtomicExpansionKind::CmpXChg;
}
- return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ llvm_unreachable("covered atomicrmw op switch");
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4e9c271197613..8497aa33449e9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16114,6 +16114,39 @@ static bool isBFloat2(Type *Ty) {
return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
}
+/// \return true if atomicrmw integer ops work for the type.
+static bool isAtomicRMWLegalIntTy(Type *Ty) {
+ if (auto *IT = dyn_cast<IntegerType>(Ty)) {
+ unsigned BW = IT->getBitWidth();
+ return BW == 32 || BW == 64;
+ }
+
+ return false;
+}
+
+/// \return true if this atomicrmw xchg type can be selected.
+static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
+ Type *Ty = RMW->getType();
+ if (isAtomicRMWLegalIntTy(Ty))
+ return true;
+
+ if (PointerType *PT = dyn_cast<PointerType>(Ty)) {
+ const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
+ unsigned BW = DL.getPointerSizeInBits(PT->getAddressSpace());
+ return BW == 32 || BW == 64;
+ }
+
+ if (Ty->isFloatTy() || Ty->isDoubleTy())
+ return true;
+
+ if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty)) {
+ return VT->getNumElements() == 2 &&
+ VT->getElementType()->getPrimitiveSizeInBits() == 16;
+ }
+
+ return false;
+}
+
/// \returns true if it's valid to emit a native instruction for \p RMW, based
/// on the properties of the target memory.
static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
@@ -16142,6 +16175,14 @@ static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
.getValueAsBool();
}
+/// \return Action to perform on AtomicRMWInsts for integer operations.
+static TargetLowering::AtomicExpansionKind
+atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
+ return isAtomicRMWLegalIntTy(RMW->getType())
+ ? TargetLowering::AtomicExpansionKind::None
+ : TargetLowering::AtomicExpansionKind::CmpXChg;
+}
+
TargetLowering::AtomicExpansionKind
SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
unsigned AS = RMW->getPointerAddressSpace();
@@ -16161,7 +16202,22 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
SSID == SyncScope::System ||
SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
- switch (RMW->getOperation()) {
+ auto Op = RMW->getOperation();
+ switch (Op) {
+ case AtomicRMWInst::Xchg: {
+ // PCIe supports add and xchg for system atomics.
+ return isAtomicRMWLegalXChgTy(RMW)
+ ? TargetLowering::AtomicExpansionKind::None
+ : TargetLowering::AtomicExpansionKind::CmpXChg;
+
+ // PCIe supports add and xchg for system atomics.
+ return atomicSupportedIfLegalIntType(RMW);
+ }
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::And:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
+ return atomicSupportedIfLegalIntType(RMW);
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor: {
@@ -16173,7 +16229,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
return AtomicExpansionKind::Expand;
}
- break;
+ return atomicSupportedIfLegalIntType(RMW);
}
case AtomicRMWInst::FAdd: {
Type *Ty = RMW->getType();
@@ -16335,13 +16391,16 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
if (HasSystemScope)
return AtomicExpansionKind::CmpXChg;
}
- break;
+
+ return atomicSupportedIfLegalIntType(RMW);
}
+ case AtomicRMWInst::Nand:
+ case AtomicRMWInst::FSub:
default:
- break;
+ return AtomicExpansionKind::CmpXChg;
}
- return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
+ llvm_unreachable("covered atomicrmw op switch");
}
TargetLowering::AtomicExpansionKind
More information about the llvm-commits
mailing list