[llvm] [SDAG][NVPTX] Support cache hints on atomic operations (PR #214918)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 19:56:51 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-nvptx
@llvm/pr-subscribers-llvm-selectiondag
Author: Yonah Goldberg (YonahGoldberg)
<details>
<summary>Changes</summary>
Support cache hints on atomic loads/stores, these have the same support as normal loads/stores.
Support cache hints on atomicrmw. These will drop everything except for .level::cache_hint.
Cmpxchg doesn't support cache hints, should drop all.
---
Patch is 48.76 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214918.diff
5 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (+13-4)
- (modified) llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp (+75-41)
- (modified) llvm/lib/Target/NVPTX/NVPTXIntrinsics.td (+73-38)
- (modified) llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll (+44-14)
- (modified) llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll (+217-15)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a2b9a14ed9543..df0c442781a7d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5254,9 +5254,11 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
MachineFunction &MF = DAG.getMachineFunction();
+ const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(), SSID, SuccessOrdering, FailureOrdering);
+ I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
+ SSID, SuccessOrdering, FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
dl, MemVT, VTs, InChain,
@@ -5325,9 +5327,11 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
MachineFunction &MF = DAG.getMachineFunction();
+ const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(), SSID, Ordering);
+ I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
+ SSID, Ordering);
SDValue L =
DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -5372,9 +5376,11 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
const MDNode *Ranges = getRangeMetadata(I);
+ const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(AAMDNodes(), Ranges), SSID, Order);
+ I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
+ Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -5409,9 +5415,12 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
MachineFunction &MF = DAG.getMachineFunction();
+ const MDNode *MemCacheHint =
+ getMemCacheHintMetadata(I, I.getPointerOperandIndex());
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(), SSID, Ordering);
+ I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
+ SSID, Ordering);
SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index b3bb96c202c55..c26c943913ccb 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -72,14 +72,21 @@ struct NVPTXScopes {
LLVMContext *Context = nullptr;
};
+enum class NVPTXMemCacheHintInstruction { Ld, St, Atom };
+
struct NVPTXMemCacheHintAccess {
+ NVPTXMemCacheHintInstruction Instruction;
NVPTX::AddressSpace AddrSpace;
- bool IsLoad;
unsigned NumElts;
unsigned EltWidth;
bool IsVolatile;
};
+struct NVPTXMemCacheHintOperands {
+ SDValue EvictionAndPrefetchHint;
+ SDValue CachePolicyReg;
+};
+
class NVPTXDAGToDAGISel : public SelectionDAGISel {
const NVPTXTargetMachine &TM;
@@ -149,7 +156,7 @@ class NVPTXDAGToDAGISel : public SelectionDAGISel {
// dropped. If L2::cache_hint is active, returns the hint with
// L2CacheHintBit set and a register containing the 64-bit cache policy
// value. Otherwise returns NOREG for the policy operand.
- std::pair<unsigned, SDValue>
+ NVPTXMemCacheHintOperands
getMemCacheHintOperands(const MemSDNode *N, NVPTXMemCacheHintAccess Access,
const SDLoc &DL);
@@ -1196,6 +1203,21 @@ static bool isGlobalOrGeneric(NVPTX::AddressSpace AddrSpace) {
AddrSpace == NVPTX::AddressSpace::Generic;
}
+static bool isLdOrSt(NVPTXMemCacheHintAccess Access) {
+ return Access.Instruction == NVPTXMemCacheHintInstruction::Ld ||
+ Access.Instruction == NVPTXMemCacheHintInstruction::St;
+}
+
+static bool isL1EvictionSupported(const NVPTXSubtarget &Subtarget,
+ NVPTX::L1Eviction Eviction,
+ NVPTXMemCacheHintAccess Access) {
+ if (Eviction == NVPTX::L1Eviction::Normal)
+ return true;
+
+ return isLdOrSt(Access) && !Access.IsVolatile &&
+ Subtarget.hasL1EvictionHint();
+}
+
static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
NVPTX::L2Prefetch Prefetch,
NVPTXMemCacheHintAccess Access) {
@@ -1203,14 +1225,14 @@ static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
case NVPTX::L2Prefetch::None:
return true;
case NVPTX::L2Prefetch::Bytes64:
- return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
- Subtarget.hasL2Prefetch64B();
+ return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
+ isGlobalOrGeneric(Access.AddrSpace) && Subtarget.hasL2Prefetch64B();
case NVPTX::L2Prefetch::Bytes128:
- return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
- Subtarget.hasL2Prefetch128B();
+ return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
+ isGlobalOrGeneric(Access.AddrSpace) && Subtarget.hasL2Prefetch128B();
case NVPTX::L2Prefetch::Bytes256:
- return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
- Subtarget.hasL2Prefetch256B();
+ return Access.Instruction == NVPTXMemCacheHintInstruction::Ld &&
+ isGlobalOrGeneric(Access.AddrSpace) && Subtarget.hasL2Prefetch256B();
}
llvm_unreachable("Unexpected L2 prefetch hint");
}
@@ -1221,22 +1243,28 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L2Eviction::Normal)
return true;
- return Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(Access.AddrSpace) &&
- !Access.IsVolatile &&
+ return isLdOrSt(Access) && !Access.IsVolatile &&
+ Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(Access.AddrSpace) &&
((Access.NumElts == 8 && Access.EltWidth == 32) ||
(Access.NumElts == 4 && Access.EltWidth == 64));
}
-std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
+static bool isCachePolicySupported(const NVPTXSubtarget &Subtarget,
+ NVPTXMemCacheHintAccess Access) {
+ return !Access.IsVolatile && isGlobalOrGeneric(Access.AddrSpace) &&
+ Subtarget.hasL2CacheHint();
+}
+
+NVPTXMemCacheHintOperands NVPTXDAGToDAGISel::getMemCacheHintOperands(
const MemSDNode *N, NVPTXMemCacheHintAccess Access, const SDLoc &DL) {
LLVMContext &Ctx = *CurDAG->getContext();
const MDNode *Node = N->getMemCacheHint();
SDValue PolicyReg = CurDAG->getRegister(NVPTX::NoRegister, MVT::i64);
if (!Node)
- return {0, PolicyReg};
+ return {getI32Imm(0, DL), PolicyReg};
if (Node->getNumOperands() == 0) {
emitInvalidMemCacheHint(Ctx, "empty hint node");
- return {0, PolicyReg};
+ return {getI32Imm(0, DL), PolicyReg};
}
NVPTX::L1Eviction L1 = NVPTX::L1Eviction::Normal;
@@ -1252,7 +1280,7 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
if (KeyStr == "nvvm.l1_eviction") {
auto ParsedL1 =
parseMemCacheHintStringValue(Ctx, KeyStr, Value, parseL1Eviction);
- if (ParsedL1 && !Access.IsVolatile && Subtarget->hasL1EvictionHint())
+ if (ParsedL1 && isL1EvictionSupported(*Subtarget, *ParsedL1, Access))
L1 = *ParsedL1;
continue;
}
@@ -1279,8 +1307,7 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
if (!ValCI)
emitInvalidMemCacheHint(
Ctx, "'nvvm.l2_cache_hint' expects an integer value");
- else if (isGlobalOrGeneric(Access.AddrSpace) && !Access.IsVolatile &&
- Subtarget->hasL2CacheHint())
+ else if (isCachePolicySupported(*Subtarget, Access))
CachePolicy = ValCI->getZExtValue();
continue;
}
@@ -1297,7 +1324,7 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
Bitfield::set<NVPTX::L2CacheHintBit>(EvictionAndPrefetchHint, true);
}
- return {EvictionAndPrefetchHint, PolicyReg};
+ return {getI32Imm(EvictionAndPrefetchHint, DL), PolicyReg};
}
bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
@@ -1345,7 +1372,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
LD,
- {CodeAddrSpace, /*IsLoad=*/true,
+ {NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
/*NumElts=*/1, /*EltWidth=*/FromTypeWidth, LD->isVolatile()},
DL);
@@ -1358,7 +1385,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
- getI32Imm(EvictionAndPrefetchHint, DL),
+ EvictionAndPrefetchHint,
PolicyReg,
Chain};
@@ -1425,7 +1452,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
const auto [EvictionAndPrefetchHint, PolicyReg] =
getMemCacheHintOperands(LD,
- {CodeAddrSpace, /*IsLoad=*/true,
+ {NVPTXMemCacheHintInstruction::Ld, CodeAddrSpace,
/*NumElts=*/LD->getNumValues() - 1,
/*EltWidth=*/FromTypeWidth, LD->isVolatile()},
DL);
@@ -1438,7 +1465,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
- getI32Imm(EvictionAndPrefetchHint, DL),
+ EvictionAndPrefetchHint,
PolicyReg,
Chain};
@@ -1493,18 +1520,17 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
ExtensionType != ISD::NON_EXTLOAD));
const auto [Base, Offset] = selectADDR(LD->getOperand(1), CurDAG);
- const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(LD,
- {NVPTX::AddressSpace::Global,
- /*IsLoad=*/true, LD->getNumValues() - 1,
- FromTypeWidth, LD->isVolatile()},
- DL);
+ const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
+ LD,
+ {NVPTXMemCacheHintInstruction::Ld, NVPTX::AddressSpace::Global,
+ LD->getNumValues() - 1, FromTypeWidth, LD->isVolatile()},
+ DL);
SDValue Ops[] = {getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL),
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
- getI32Imm(EvictionAndPrefetchHint, DL),
+ EvictionAndPrefetchHint,
PolicyReg,
LD->getChain()};
@@ -1618,7 +1644,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
// Extract eviction/prefetch hint and cache policy register.
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
- {CodeAddrSpace, /*IsLoad=*/false,
+ {NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
/*NumElts=*/1, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
DL);
@@ -1629,7 +1655,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
getI32Imm(ToTypeWidth, DL),
Base,
Offset,
- getI32Imm(EvictionAndPrefetchHint, DL),
+ EvictionAndPrefetchHint,
PolicyReg,
Chain};
@@ -1679,15 +1705,14 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
// Extract eviction/prefetch hint and cache policy register.
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
ST,
- {CodeAddrSpace, /*IsLoad=*/false, /*NumElts=*/NumElts,
- /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
+ {NVPTXMemCacheHintInstruction::St, CodeAddrSpace,
+ /*NumElts=*/NumElts, /*EltWidth=*/ToTypeWidth, ST->isVolatile()},
DL);
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
Ops.append({getI32Imm(Ordering, DL), getI32Imm(Scope, DL),
getI32Imm(CodeAddrSpace, DL), getI32Imm(ToTypeWidth, DL), Base,
- Offset, getI32Imm(EvictionAndPrefetchHint, DL), PolicyReg,
- Chain});
+ Offset, EvictionAndPrefetchHint, PolicyReg, Chain});
const MVT::SimpleValueType EltVT =
ST->getOperand(1).getSimpleValueType().SimpleTy;
@@ -2308,14 +2333,23 @@ void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) {
const SDValue Chain = N->getOperand(0);
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
- SmallVector<SDValue, 5> Ops{Base, Offset};
+ SmallVector<SDValue, 10> Ops{Base, Offset};
Ops.append(N->op_begin() + 2, N->op_end());
- Ops.append({
- getI32Imm(getMemOrder(AN), dl),
- getI32Imm(getAtomicScope(AN), dl),
- getI32Imm(getAddrSpace(AN), dl),
- Chain,
- });
+ Ops.append({getI32Imm(getMemOrder(AN), dl), getI32Imm(getAtomicScope(AN), dl),
+ getI32Imm(getAddrSpace(AN), dl)});
+
+ if (N->getOpcode() == NVPTXISD::ATOMIC_SWAP_B128) {
+ unsigned EltWidth = AN->getMemoryVT().getFixedSizeInBits();
+ NVPTXMemCacheHintAccess Access{NVPTXMemCacheHintInstruction::Atom,
+ getAddrSpace(AN),
+ /*NumElts=*/1, EltWidth, AN->isVolatile()};
+ const auto [EvictionAndPrefetchHint, CachePolicyReg] =
+ getMemCacheHintOperands(AN, Access, dl);
+ Ops.push_back(EvictionAndPrefetchHint);
+ Ops.push_back(CachePolicyReg);
+ }
+
+ Ops.push_back(Chain);
assert(N->getOpcode() == NVPTXISD::ATOMIC_CMP_SWAP_B128 ||
N->getOpcode() == NVPTXISD::ATOMIC_SWAP_B128);
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 83caf03942b26..2e11871cefa1d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2527,32 +2527,65 @@ class AtomicSDNodeXForm<SDNode atomic> {
SDNodeXForm GetAddSp = SDNodeXForm<atomic, [{
return getI32Imm(getAddrSpace(cast<MemSDNode>(N)), SDLoc(N));
}]>;
+
+ SDNodeXForm GetEvictionAndPrefetchHint = SDNodeXForm<atomic, [{
+ auto *MN = cast<MemSDNode>(N);
+ unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
+ NVPTXMemCacheHintAccess Access{
+ NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
+ /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ return getMemCacheHintOperands(MN, Access, SDLoc(N))
+ .EvictionAndPrefetchHint;
+ }]>;
+
+ SDNodeXForm GetCachePolicy = SDNodeXForm<atomic, [{
+ auto *MN = cast<MemSDNode>(N);
+ unsigned EltWidth = MN->getMemoryVT().getFixedSizeInBits();
+ NVPTXMemCacheHintAccess Access{
+ NVPTXMemCacheHintInstruction::Atom, getAddrSpace(MN),
+ /*NumElts=*/1, EltWidth, MN->isVolatile()};
+ return getMemCacheHintOperands(MN, Access, SDLoc(N)).CachePolicyReg;
+ }]>;
}
-multiclass F_ATOMIC_2<RegTyInfo t, SDPatternOperator op, string op_str, SDNode atomic, list<Predicate> preds = []> {
- defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}" # "." # op_str;
+multiclass F_ATOMIC_2<RegTyInfo t, SDPatternOperator op, string op_str,
+ string type_str, SDNode atomic,
+ list<Predicate> preds = []> {
+ defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}." # op_str #
+ "${evictionAndPrefetchHint:l2}." # type_str #
+ " \t$dst, [$addr], $b${policy};";
let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
- def _r : BasicFlagsNVPTXInst<(outs t.RC:$dst),
- (ins ADDR:$addr, t.RC:$b),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),
- asm_str>,
+ def _r : NVPTXInst<
+ (outs t.RC:$dst),
+ (ins ADDR:$addr, t.RC:$b, AtomicCode:$sem, AtomicCode:$scope,
+ AtomicCode:$addsp,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy), asm_str>,
Requires<preds>;
if t.SupportsImm then
- def _i : BasicFlagsNVPTXInst<(outs t.RC:$dst),
- (ins ADDR:$addr, t.Imm:$b),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),
- asm_str>,
+ def _i : NVPTXInst<
+ (outs t.RC:$dst),
+ (ins ADDR:$addr, t.Imm:$b, AtomicCode:$sem, AtomicCode:$scope,
+ AtomicCode:$addsp,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy), asm_str>,
Requires<preds>;
}
defvar XForm = AtomicSDNodeXForm<atomic>;
def : Pat<(op:$this addr:$addr, t.Ty:$b),
- (!cast<Instruction>(NAME # _r) ADDR:$addr, t.Ty:$b, (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _r) ADDR:$addr, t.Ty:$b, (XForm.GetSem $this),
+ (XForm.GetScope $this), (XForm.GetAddSp $this),
+ (XForm.GetEvictionAndPrefetchHint $this),
+ (XForm.GetCachePolicy $this))>;
if t.SupportsImm then
def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b)),
- (!cast<Instruction>(NAME # _i) ADDR:$addr, (t.Ty t.ImmNode:$b), (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _i) ADDR:$addr, (t.Ty t.ImmNode:$b),
+ (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this),
+ (XForm.GetEvictionAndPrefetchHint $this),
+ (XForm.GetCachePolicy $this))>;
}
multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode atomic> {
@@ -2598,29 +2631,29 @@ multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode a
defm atomic_load_fadd : binary_atomic_op_fp<atomic_load_fadd>;
// atom_add
-defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2<I32RT, atomic_load_add_i32, "add.u32", atomic_load_add>;
-defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add.u64", atomic_load_add>;
+defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2<I32RT, atomic_load_add_i32, "add", "u32", atomic_load_add>;
+defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add", "u64", atomic_load_add>;
-defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz.f16", atomic_load_fadd, [SM70, hasPTX<63>]>;
-defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz.bf16", atomic_load_fadd, [SM90, hasPTX<78>]>;
-defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add.f32", atomic_load_fadd>;
-defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add.f64", atomic_load_fadd, [hasAtomAddF64]>;
+defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz", "f16", atomic_load_fadd, [SM70, hasPTX<63>]>;
+defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90, hasPTX<78>]>;
+defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add", "f32", atomic_load_fadd>;
+defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add", "f64", atomic_load_fadd, [hasAtomAddF64]>;
// atom_swap
-defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2<I32RT, atomic_swap_i32, "exch.b32", atomic_swap>;
-defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2<I64RT, atomic_swap_i64, "exch.b64", atomic_swap>;
+defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2<I32RT, atomic_swap_i32, "exch", "b32", atomic_swap>;
+defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2<I64RT, atomic_swap_i64, "exch", "b64", atomic_swap>;
// atom_max
-defm INT_PTX_ATOMIC_MAX_32 : F_ATOMIC_2<I32RT, atomic_load_max_i32, "max.s32", atomic_load_m...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214918
More information about the llvm-commits
mailing list