[llvm] Revert "[NVPTX][AtomicExpandPass] Complete support for AtomicRMW in NVPTX (#176015)" (PR #178329)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 27 16:56:55 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-nvptx
Author: Akshay Deodhar (akshayrdeodhar)
<details>
<summary>Changes</summary>
This reverts commit 1d379d05d46d77b5f008349cc14de27dd055f4b9. This change breaks llvm-nvptx-nvidia-win in the buildbot.
---
Patch is 442.81 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/178329.diff
14 Files Affected:
- (modified) llvm/lib/CodeGen/AtomicExpandPass.cpp (+1-4)
- (modified) llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp (+35-79)
- (modified) llvm/lib/Target/NVPTX/NVPTXIntrinsics.td (+60-84)
- (added) llvm/test/CodeGen/NVPTX/atomicrmw-expand.ll (+151)
- (removed) llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll (-3137)
- (removed) llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll (-3111)
- (removed) llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll (-2983)
- (removed) llvm/test/CodeGen/NVPTX/atomicrmw.py (-116)
- (modified) llvm/test/CodeGen/NVPTX/atomics-sm60.ll (+14-3)
- (added) llvm/test/CodeGen/NVPTX/atomics-sm70.ll (+144)
- (added) llvm/test/CodeGen/NVPTX/atomics-sm90.ll (+147)
- (modified) llvm/test/CodeGen/NVPTX/atomics.ll (+476-3)
- (modified) llvm/test/CodeGen/NVPTX/distributed-shared-cluster.ll (+24-28)
- (modified) llvm/test/CodeGen/NVPTX/lit.local.cfg (+1-1)
``````````diff
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 6182c3bbd6a73..3d2a52dab153e 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -331,7 +331,7 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
} else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
isAcquireOrStronger(RMWI->getOrdering()))) {
FenceOrdering = RMWI->getOrdering();
- RMWI->setOrdering(TLI->atomicOperationOrderAfterFenceSplit(RMWI));
+ RMWI->setOrdering(AtomicOrdering::Monotonic);
} else if (CASI &&
TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
TargetLoweringBase::AtomicExpansionKind::None &&
@@ -1681,9 +1681,6 @@ Value *AtomicExpandImpl::insertRMWCmpXchgLoop(
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);
LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
- // TODO: The initial load must be atomic with the same synchronization scope
- // to avoid a data race with concurrent stores. If the instruction being
- // emulated is volatile, issue a volatile load.
Builder.CreateBr(LoopBB);
// Start the main loop block now that we've taken care of the preliminaries.
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 8da48b414dcbf..1be35a1c67457 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7382,84 +7382,46 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
- // This function returns true iff the operation is emulated using a CAS-loop,
- // or if it has the memory order seq_cst (which is not natively supported in
- // the PTX `atom` instruction).
- //
- // atomicrmw and cmpxchg instructions not efficiently supported by PTX
- // are lowered to CAS emulation loops that preserve their memory order,
- // syncscope, and volatile semantics. For PTX, it is more efficient to use
- // atom.cas.relaxed.sco instructions within the loop, and fences before and
- // after the loop to restore order.
- //
- // Atomic instructions efficiently supported by PTX are lowered to
- // `atom.<op>.<sem>.<scope` instruction with their corresponding memory order
- // and scope. Since PTX does not support seq_cst, we emulate it by lowering to
- // a fence.sc followed by an atom according to the PTX atomics ABI
- // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
- if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I))
- return (cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
- CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
- if (auto *RI = dyn_cast<AtomicRMWInst>(I))
- return shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg ||
- RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
- return false;
+ auto *CI = dyn_cast<AtomicCmpXchgInst>(I);
+ // When CAS bitwidth is not supported on the hardware, the CAS is emulated
+ // using a retry loop that uses a higher-bitwidth monotonic CAS. We enforce
+ // the memory order using explicit fences around the retry loop.
+ // The memory order of natively supported CAS operations can be enforced
+ // by lowering to an atom.cas with the right memory synchronizing effect.
+ // However, atom.cas only supports relaxed, acquire, release and acq_rel.
+ // So we also use explicit fences for enforcing memory order for
+ // seq_cast CAS with natively-supported bitwidths.
+ return CI &&
+ (cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <
+ STI.getMinCmpXchgSizeInBits() ||
+ CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent);
}
AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
const Instruction *I) const {
- // If the operation is emulated by a CAS-loop, we lower the instruction to
- // atom.<op>.relaxed, since AtomicExpandPass will insert fences for enforcing
- // the correct memory ordering around the CAS loop.
- //
- // When the operation is not emulated, but the memory order is seq_cst,
- // we must lower to "fence.sc.<scope>; atom.<op>.acquire.<scope>;" to conform
- // to the PTX atomics ABI.
- // https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
- // For such cases, emitLeadingFence() will separately insert the leading
- // "fence.sc.<scope>;". Here, we only set the memory order to acquire.
- //
- // Otherwise, the operation is not emulated, and the memory order is not
- // seq_cst. In this case, the LLVM memory order is natively supported by the
- // PTX `atom` instruction, and we just lower to the corresponding
- // `atom.<op>.relaxed|acquire|release|acq_rel". For such cases, this function
- // will NOT be called.
- // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
- // I before its memory order was modified.
- if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I);
+ auto *CI = dyn_cast<AtomicCmpXchgInst>(I);
+ bool BitwidthSupportedAndIsSeqCst =
CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=
- STI.getMinCmpXchgSizeInBits())
- return AtomicOrdering::Acquire;
- else if (auto *RI = dyn_cast<AtomicRMWInst>(I);
- RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
- shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
- return AtomicOrdering::Acquire;
-
- return AtomicOrdering::Monotonic;
+ STI.getMinCmpXchgSizeInBits();
+ return BitwidthSupportedAndIsSeqCst ? AtomicOrdering::Acquire
+ : AtomicOrdering::Monotonic;
}
Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
- // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
- // `Inst` before its memory order was modified. We cannot enforce this with an
- // assert, because AtomicExpandPass will have modified the memory order
- // between the initial call to shouldInsertFencesForAtomic() and the call to
- // this function.
- if (!isa<AtomicCmpXchgInst>(Inst) && !isa<AtomicRMWInst>(Inst))
+ if (!isa<AtomicCmpXchgInst>(Inst))
return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
- // Specialize for cmpxchg and atomicrmw
- auto SSID = getAtomicSyncScopeID(Inst);
- assert(SSID.has_value() && "Expected an atomic operation");
-
+ // Specialize for cmpxchg
+ // Emit a fence.sc leading fence for cmpxchg seq_cst which are not emulated
+ SyncScope::ID SSID = cast<AtomicCmpXchgInst>(Inst)->getSyncScopeID();
if (isReleaseOrStronger(Ord))
return Builder.CreateFence(Ord == AtomicOrdering::SequentiallyConsistent
- ? AtomicOrdering::SequentiallyConsistent
+ ? Ord
: AtomicOrdering::Release,
- SSID.value());
+ SSID);
return nullptr;
}
@@ -7467,25 +7429,19 @@ Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
- // prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
- // `Inst` before its memory order was modified. See `emitLeadingFence` for why
- // this cannot be enforced with an assert. Specialize for cmpxchg and
- // atomicrmw
- auto *CI = dyn_cast<AtomicCmpXchgInst>(Inst);
- auto *RI = dyn_cast<AtomicRMWInst>(Inst);
- if (!CI && !RI)
+ // Specialize for cmpxchg
+ if (!isa<AtomicCmpXchgInst>(Inst))
return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
- auto SSID = getAtomicSyncScopeID(Inst);
- assert(SSID.has_value() && "Expected an atomic operation");
-
- bool IsEmulated =
- CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
- : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
-
- if (isAcquireOrStronger(Ord) && IsEmulated)
- return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
+ auto *CI = cast<AtomicCmpXchgInst>(Inst);
+ auto CASWidth =
+ cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth();
+ SyncScope::ID SSID = CI->getSyncScopeID();
+ // Do not emit a trailing fence for cmpxchg seq_cst which are not emulated
+ if (isAcquireOrStronger(Ord) &&
+ (Ord != AtomicOrdering::SequentiallyConsistent ||
+ CASWidth < STI.getMinCmpXchgSizeInBits()))
+ return Builder.CreateFence(AtomicOrdering::Acquire, SSID);
return nullptr;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 13ae74740c3e1..ad5dd356ee90f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2372,7 +2372,16 @@ def INT_FNS_iii : INT_FNS_MBO<(ins i32imm:$mask, i32imm:$base, i32imm:$
// Atomic Functions
//-----------------------------------
-multiclass F_ATOMIC_2_INTRINSIC<RegTyInfo t, string sem_str, string as_str, string op_str,
+class ATOMIC_GLOBAL_CHK <dag frag>
+ : PatFrag<!setdagop(frag, ops), frag, AS_match.global>;
+class ATOMIC_SHARED_CHK <dag frag>
+ : PatFrag<!setdagop(frag, ops), frag, AS_match.shared>;
+class ATOMIC_SHARED_CLUSTER_CHK <dag frag>
+ : PatFrag<!setdagop(frag, ops), frag, AS_match.shared_cluster>;
+class ATOMIC_GENERIC_CHK <dag frag>
+ : PatFrag<!setdagop(frag, ops), frag, AS_match.generic>;
+
+multiclass F_ATOMIC_2<RegTyInfo t, string sem_str, string as_str, string op_str,
SDPatternOperator op, list<Predicate> preds> {
defvar asm_str = "atom" # sem_str # as_str # "." # op_str;
let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
@@ -2388,48 +2397,8 @@ multiclass F_ATOMIC_2_INTRINSIC<RegTyInfo t, string sem_str, string as_str, stri
}
}
-class AtomicSDNodeXForm<SDNode atomic> {
- SDNodeXForm GetSem = SDNodeXForm<atomic, [{
- return getI32Imm(getMemOrder(cast<MemSDNode>(N)), SDLoc(N));
- }]>;
-
- SDNodeXForm GetScope = SDNodeXForm<atomic, [{
- return getI32Imm(getAtomicScope(cast<MemSDNode>(N)), SDLoc(N));
- }]>;
-
- SDNodeXForm GetAddSp = SDNodeXForm<atomic, [{
- return getI32Imm(getAddrSpace(cast<MemSDNode>(N)), SDLoc(N));
- }]>;
-}
-
-multiclass F_ATOMIC_2<RegTyInfo t, SDPatternOperator op, string op_str, SDNode atomic, list<Predicate> preds = []> {
- defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}" # "." # op_str;
- let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
- def _r : BasicFlagsNVPTXInst<(outs t.RC:$dst),
- (ins ADDR:$addr, t.RC:$b),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),
- asm_str>,
- Requires<preds>;
- if t.SupportsImm then
- def _i : BasicFlagsNVPTXInst<(outs t.RC:$dst),
- (ins ADDR:$addr, t.Imm:$b),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp),
- asm_str>,
- Requires<preds>;
- }
-
- defvar XForm = AtomicSDNodeXForm<atomic>;
-
- def : Pat<(op:$this addr:$addr, t.Ty:$b),
- (!cast<Instruction>(NAME # _r) ADDR:$addr, t.Ty:$b, (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
-
- if t.SupportsImm then
- def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b)),
- (!cast<Instruction>(NAME # _i) ADDR:$addr, (t.Ty t.ImmNode:$b), (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
-}
-
multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode atomic> {
- defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}" # "." # op_str;
+ defvar asm_str = "atom${sem:sem}${scope:scope}${addsp:addsp}" # op_str;
let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
def _rr : BasicFlagsNVPTXInst<(outs t.RC:$dst),
@@ -2453,80 +2422,87 @@ multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode a
asm_str>;
}
- defvar XForm = AtomicSDNodeXForm<atomic>;
+ defvar GetSem = SDNodeXForm<atomic, [{
+ return getI32Imm(getMemOrder(cast<MemSDNode>(N)), SDLoc(N));
+ }]>;
+
+ defvar GetScope = SDNodeXForm<atomic, [{
+ return getI32Imm(getAtomicScope(cast<MemSDNode>(N)), SDLoc(N));
+ }]>;
+
+ defvar GetAddSp = SDNodeXForm<atomic, [{
+ return getI32Imm(getAddrSpace(cast<MemSDNode>(N)), SDLoc(N));
+ }]>;
def : Pat<(op:$this addr:$addr, t.Ty:$b, t.Ty:$c),
- (!cast<Instruction>(NAME # _rr) ADDR:$addr, t.Ty:$b, t.Ty:$c, (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _rr) ADDR:$addr, t.Ty:$b, t.Ty:$c, (GetSem $this), (GetScope $this), (GetAddSp $this))>;
def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c),
- (!cast<Instruction>(NAME # _ir) ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _ir) ADDR:$addr, (t.Ty t.ImmNode:$b), t.Ty:$c, (GetSem $this), (GetScope $this), (GetAddSp $this))>;
def : Pat<(op:$this addr:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c)),
- (!cast<Instruction>(NAME # _ri) ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _ri) ADDR:$addr, t.Ty:$b, (t.Ty t.ImmNode:$c), (GetSem $this), (GetScope $this), (GetAddSp $this))>;
def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c)),
- (!cast<Instruction>(NAME # _ii) ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), (XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this))>;
+ (!cast<Instruction>(NAME # _ii) ADDR:$addr, (t.Ty t.ImmNode:$b), (t.Ty t.ImmNode:$c), (GetSem $this), (GetScope $this), (GetAddSp $this))>;
}
-defm atomic_load_fadd : binary_atomic_op_fp<atomic_load_fadd>;
+multiclass F_ATOMIC_2_AS<RegTyInfo t, SDPatternOperator frag, string op_str, list<Predicate> preds = []> {
+ defvar frag_pat = (frag node:$a, node:$b);
+ defm _G : F_ATOMIC_2<t, "", ".global", op_str, ATOMIC_GLOBAL_CHK<frag_pat>, preds>;
+ defm _S : F_ATOMIC_2<t, "", ".shared", op_str, ATOMIC_SHARED_CHK<frag_pat>, preds>;
+ defm _S_C : F_ATOMIC_2<t, "", ".shared::cluster", op_str, ATOMIC_SHARED_CLUSTER_CHK<frag_pat>, !listconcat([hasClusters], preds)>;
+ defm _GEN : F_ATOMIC_2<t, "", "", op_str, ATOMIC_GENERIC_CHK<frag_pat>, preds>;
+}
// atom_add
-defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2<I32RT, atomic_load_add_i32, "add.u32", atomic_load_add>;
-defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add.u64", atomic_load_add>;
+defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2_AS<I32RT, atomic_load_add_i32, "add.u32">;
+defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2_AS<I64RT, atomic_load_add_i64, "add.u64">;
-defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz.f16", atomic_load_fadd, [hasSM<70>, hasPTX<63>]>;
-defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz.bf16", atomic_load_fadd, [hasSM<90>, hasPTX<78>]>;
-defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add.f32", atomic_load_fadd>;
-defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add.f64", atomic_load_fadd, [hasAtomAddF64]>;
+defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2_AS<F16RT, atomic_load_fadd, "add.noftz.f16", [hasSM<70>, hasPTX<63>]>;
+defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2_AS<BF16RT, atomic_load_fadd, "add.noftz.bf16", [hasSM<90>, hasPTX<78>]>;
+defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2_AS<F32RT, atomic_load_fadd, "add.f32">;
+defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2_AS<F64RT, atomic_load_fadd, "add.f64", [hasAtomAddF64]>;
// atom_swap
-defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2<I32RT, atomic_swap_i32, "exch.b32", atomic_swap>;
-defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2<I64RT, atomic_swap_i64, "exch.b64", atomic_swap>;
+defm INT_PTX_ATOM_SWAP_32 : F_ATOMIC_2_AS<I32RT, atomic_swap_i32, "exch.b32">;
+defm INT_PTX_ATOM_SWAP_64 : F_ATOMIC_2_AS<I64RT, atomic_swap_i64, "exch.b64">;
// atom_max
-defm INT_PTX_ATOMIC_MAX_32 : F_ATOMIC_2<I32RT, atomic_load_max_i32, "max.s32", atomic_load_max>;
-defm INT_PTX_ATOMIC_MAX_64 : F_ATOMIC_2<I64RT, atomic_load_max_i64, "max.s64", atomic_load_max, [hasSM<32>]>;
-defm INT_PTX_ATOMIC_UMAX_32 : F_ATOMIC_2<I32RT, atomic_load_umax_i32, "max.u32", atomic_load_umax>;
-defm INT_PTX_ATOMIC_UMAX_64 : F_ATOMIC_2<I64RT, atomic_load_umax_i64, "max.u64", atomic_load_umax, [hasSM<32>]>;
+defm INT_PTX_ATOMIC_MAX_32 : F_ATOMIC_2_AS<I32RT, atomic_load_max_i32, "max.s32">;
+defm INT_PTX_ATOMIC_MAX_64 : F_ATOMIC_2_AS<I64RT, atomic_load_max_i64, "max.s64", [hasSM<32>]>;
+defm INT_PTX_ATOMIC_UMAX_32 : F_ATOMIC_2_AS<I32RT, atomic_load_umax_i32, "max.u32">;
+defm INT_PTX_ATOMIC_UMAX_64 : F_ATOMIC_2_AS<I64RT, atomic_load_umax_i64, "max.u64", [hasSM<32>]>;
// atom_min
-defm INT_PTX_ATOMIC_MIN_32 : F_ATOMIC_2<I32RT, atomic_load_min_i32, "min.s32", atomic_load_min>;
-defm INT_PTX_ATOMIC_MIN_64 : F_ATOMIC_2<I64RT, atomic_load_min_i64, "min.s64", atomic_load_min, [hasSM<32>]>;
-defm INT_PTX_ATOMIC_UMIN_32 : F_ATOMIC_2<I32RT, atomic_load_umin_i32, "min.u32", atomic_load_umin>;
-defm INT_PTX_ATOMIC_UMIN_64 : F_ATOMIC_2<I64RT, atomic_load_umin_i64, "min.u64", atomic_load_umin, [hasSM<32>]>;
-
-// NOTE: The semantics for atomicrmw fmin (and fmax) upholds LangRef
-// requirements. The LangRef requires the semantics of fmin/fmax to follow
-// llvm.minimum/llvm.maximum[1], but allows sNaNs to be treated as qNaNs[2],
-// that is the llvm.minnum(sNaN, NUM) -> qNaN case can be handled as (qNaN, NUM)
-// -> NUM. NVPTX implements this by lowering to atom.min.f32[3], which has those
-// semantics.
-// [1] https://llvm.org/docs/LangRef.html#llvm-implementation
-// [2] https://llvm.org/docs/LangRef.html#behavior-of-floating-point-nan-values
-// [3] https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#half-precision-floating-point-instructions-min
+defm INT_PTX_ATOMIC_MIN_32 : F_ATOMIC_2_AS<I32RT, atomic_load_min_i32, "min.s32">;
+defm INT_PTX_ATOMIC_MIN_64 : F_ATOMIC_2_AS<I64RT, atomic_load_min_i64, "min.s64", [hasSM<32>]>;
+defm INT_PTX_ATOMIC_UMIN_32 : F_ATOMIC_2_AS<I32RT, atomic_load_umin_i32, "min.u32">;
+defm INT_PTX_ATOMIC_UMIN_64 : F_ATOMIC_2_AS<I64RT, atomic_load_umin_i64, "min.u64", [hasSM<32>]>;
// atom_inc atom_dec
-defm INT_PTX_ATOM_INC_32 : F_ATOMIC_2<I32RT, atomic_load_uinc_wrap_i32, "inc.u32", atomic_load_uinc_wrap>;
-defm INT_PTX_ATOM_DEC_32 : F_ATOMIC_2<I32RT, atomic_load_udec_wrap_i32, "dec.u32", atomic_load_udec_wrap>;
+defm INT_PTX_ATOM_INC_32 : F_ATOMIC_2_AS<I32RT, atomic_load_uinc_wrap_i32, "inc.u32">;
+defm INT_PTX_ATOM_DEC_32 : F_ATOMIC_2_AS<I32RT, atomic_load_udec_wrap_i32, "dec.u32">;
// atom_and
-defm INT_PTX_ATOM_AND_32 : F_ATOMIC_2<I32RT, atomic_load_and_i32, "and.b32", atomic_load_and>;
-defm INT_PTX_ATOM_AND_64 : F_ATOMIC_2<I64RT, atomic_load_and_i64, "and.b64", atomic_load_and, [hasSM<32>]>;
+defm INT_PTX_ATOM_AND_32 : F_ATOMIC_2_AS<I32RT, atomic_load_and_i32, "and.b32">;
+defm INT_PTX_ATOM_AND_64 : F_ATOMIC_2_AS<I64RT, atomic_load_and_i64, "and.b64", [hasSM<32>]>;
// atom_or
-defm INT_PTX_ATOM_OR_32 : F_ATOMIC_2<I32RT, atomic_load_or_i32, "or.b32", atomic_load_or>;
-defm INT_PTX_ATOM_OR_64 : F_ATOMIC_2<I64RT, atomic_load_or_i64, "or.b64", atomic_load_or, [hasSM<32>]>;
+defm INT_PTX_ATOM_OR_32 : F_ATOMIC_2_AS<I32RT, atomic_load_or_i32, "or.b32">;
+defm INT_PTX_ATOM_OR_64 : F_ATOMIC_2_AS<I64RT, atomic_load_or_i64, "or.b64", [hasSM<32>]>;
// atom_xor
-defm INT_PTX_ATOM_XOR_32 : F_ATOMIC_2<I32RT, atomic_load_xor_i32, "xor.b32", atomic_load_xor>;
-defm INT_PTX_ATOM_XOR_64 : F_ATOMIC_2<I64RT, atomic_load_xor_i64, "xor.b64", atomic_load_xor, [hasSM<32>]>;
+defm INT_PTX_ATOM_XOR_32 : F_ATOMIC_2_AS<I...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/178329
More information about the llvm-commits
mailing list