[llvm] [AMDGPU] Remove unsound idempotent atomicrmw to atomic load lowering (PR #218377)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 04:33:46 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
An idempotent atomicrmw still holds a slot in the modification order, so folding it to a plain load can drop synchronization the memory model requires
---
Patch is 30.55 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218377.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+5-24)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (-3)
- (modified) llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll (+223-58)
- (modified) llvm/test/CodeGen/AMDGPU/mmra.ll (+33-2)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 15053568da8dc..5836fa112b087 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20787,8 +20787,10 @@ SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
Op == AtomicRMWInst::Xor) {
// Atomic sub/or/xor do not work over PCI express, but atomic add
// does. InstCombine transforms these with 0 to or, so undo that.
+ // Sub-word types are not selectable and take the cmpxchg expansion.
if (const Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
- ConstVal && ConstVal->isNullValue())
+ ConstVal && ConstVal->isNullValue() &&
+ isAtomicRMWLegalIntTy(RMW->getType()))
return AtomicExpansionKind::CustomExpand;
}
@@ -21359,7 +21361,8 @@ void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
Op == AtomicRMWInst::Xor) {
if (const auto *ConstVal = dyn_cast<Constant>(AI->getValOperand());
- ConstVal && ConstVal->isNullValue()) {
+ ConstVal && ConstVal->isNullValue() &&
+ isAtomicRMWLegalIntTy(AI->getType())) {
// atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
AI->setOperation(AtomicRMWInst::Add);
@@ -21399,25 +21402,3 @@ void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
llvm_unreachable(
"Expand Atomic Store only handles SCRATCH -> FLAT conversion");
}
-
-LoadInst *
-SITargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
- IRBuilder<> Builder(AI);
- auto Order = AI->getOrdering();
-
- // The optimization removes store aspect of the atomicrmw. Therefore, cache
- // must be flushed if the atomic ordering had a release semantics. This is
- // not necessary a fence, a release fence just coincides to do that flush.
- // Avoid replacing of an atomicrmw with a release semantics.
- if (isReleaseOrStronger(Order))
- return nullptr;
-
- LoadInst *LI = Builder.CreateAlignedLoad(
- AI->getType(), AI->getPointerOperand(), AI->getAlign());
- LI->setAtomic(Order, AI->getSyncScopeID());
- LI->copyMetadata(*AI);
- LI->takeName(AI);
- AI->replaceAllUsesWith(LI);
- AI->eraseFromParent();
- return LI;
-}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 8ece9f279a100..a0d8ec2bbaa7d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -613,9 +613,6 @@ class SITargetLowering final : public AMDGPUTargetLowering {
void emitExpandAtomicLoad(LoadInst *LI) const override;
void emitExpandAtomicStore(StoreInst *SI) const override;
- LoadInst *
- lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
-
const TargetRegisterClass *getRegClassFor(MVT VT,
bool isDivergent) const override;
bool requiresUniformRegister(MachineFunction &MF,
diff --git a/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll b/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
index 28731eb4b355c..cab9b10a7e295 100644
--- a/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
@@ -6,12 +6,14 @@ define i32 @global_agent_monotonic_idempotent_or(ptr addrspace(1) %in) {
; GFX942-LABEL: global_agent_monotonic_idempotent_or:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc1
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_or v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_or(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("agent-one-as") monotonic, align 4
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_or(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") monotonic, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") monotonic, align 4
@@ -22,13 +24,15 @@ define i32 @global_agent_acquire_idempotent_or(ptr addrspace(1) %in) {
; GFX942-LABEL: global_agent_acquire_idempotent_or:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc1
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_or v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_acquire_idempotent_or(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("agent-one-as") acquire, align 4
+; OPT-LABEL: define i32 @global_agent_acquire_idempotent_or(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") acquire, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") acquire, align 4
@@ -45,9 +49,10 @@ define i32 @global_agent_release_idempotent_or(ptr addrspace(1) %in) {
; GFX942-NEXT: global_atomic_or v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_release_idempotent_or(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4
+; OPT-LABEL: define i32 @global_agent_release_idempotent_or(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") release, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") release, align 4
@@ -64,9 +69,10 @@ define i32 @global_agent_release_idempotent_or_no_remote(ptr addrspace(1) %in) {
; GFX942-NEXT: global_atomic_or v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_release_idempotent_or_no_remote(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.remote.memory [[META0:![0-9]+]]
+; OPT-LABEL: define i32 @global_agent_release_idempotent_or_no_remote(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.remote.memory [[META0:![0-9]+]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.remote.memory !0
@@ -83,9 +89,10 @@ define i32 @global_agent_release_idempotent_or_no_fine_grained(ptr addrspace(1)
; GFX942-NEXT: global_atomic_or v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_release_idempotent_or_no_fine_grained(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_agent_release_idempotent_or_no_fine_grained(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.fine.grained.memory !0
@@ -103,9 +110,10 @@ define i32 @global_agent_acquire_release_idempotent_or(ptr addrspace(1) %in) {
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_acquire_release_idempotent_or(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4
+; OPT-LABEL: define i32 @global_agent_acquire_release_idempotent_or(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") acq_rel, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") acq_rel, align 4
@@ -123,9 +131,10 @@ define i32 @global_agent_acquire_release_idempotent_or__no_fine_grained(ptr addr
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_acquire_release_idempotent_or__no_fine_grained(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_agent_acquire_release_idempotent_or__no_fine_grained(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory !0
@@ -143,9 +152,10 @@ define i32 @global_agent_seq_cst_idempotent_or(ptr addrspace(1) %in) {
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_seq_cst_idempotent_or(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") seq_cst, align 4
+; OPT-LABEL: define i32 @global_agent_seq_cst_idempotent_or(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("agent-one-as") seq_cst, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") seq_cst, align 4
@@ -156,12 +166,14 @@ define i32 @global_agent_monotonic_idempotent_add(ptr addrspace(1) %in) {
; GFX942-LABEL: global_agent_monotonic_idempotent_add:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_add v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_add(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("workgroup") monotonic, align 4
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_add(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("workgroup") monotonic, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw add ptr addrspace(1) %in, i32 0 syncscope("workgroup") monotonic, align 4
@@ -172,12 +184,14 @@ define i32 @global_agent_monotonic_idempotent_add__no_fine_grained(ptr addrspace
; GFX942-LABEL: global_agent_monotonic_idempotent_add__no_fine_grained:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_add v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_add__no_fine_grained(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("workgroup") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_add__no_fine_grained(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("workgroup") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw add ptr addrspace(1) %in, i32 0 syncscope("workgroup") monotonic, align 4, !amdgpu.no.fine.grained.memory !0
@@ -188,12 +202,14 @@ define i32 @global_agent_monotonic_idempotent_sub(ptr addrspace(1) %in) {
; GFX942-LABEL: global_agent_monotonic_idempotent_sub:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_sub v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_sub(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("wavefront") monotonic, align 4
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_sub(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 syncscope("wavefront") monotonic, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw sub ptr addrspace(1) %in, i32 0 syncscope("wavefront") monotonic, align 4
@@ -204,12 +220,14 @@ define i32 @global_agent_monotonic_idempotent_sub__no_fine_grained(ptr addrspace
; GFX942-LABEL: global_agent_monotonic_idempotent_sub__no_fine_grained:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_sub v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_sub__no_fine_grained(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_sub__no_fine_grained(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw sub ptr addrspace(1) [[IN]], i32 0 syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw sub ptr addrspace(1) %in, i32 0 syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory !0
@@ -220,12 +238,14 @@ define i32 @global_system_monotonic_idempotent_xor(ptr addrspace(1) %in) {
; GFX942-LABEL: global_system_monotonic_idempotent_xor:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_add v0, v[0:1], v2, off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_system_monotonic_idempotent_xor(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] monotonic, align 4
+; OPT-LABEL: define i32 @global_system_monotonic_idempotent_xor(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN]], i32 0 monotonic, align 4
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw xor ptr addrspace(1) %in, i32 0 monotonic, align 4
@@ -236,12 +256,14 @@ define i32 @global_system_monotonic_idempotent_xor__no_fine_grained(ptr addrspac
; GFX942-LABEL: global_system_monotonic_idempotent_xor__no_fine_grained:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off sc0 sc1
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: global_atomic_xor v0, v[0:1], v2, off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_system_monotonic_idempotent_xor__no_fine_grained(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_system_monotonic_idempotent_xor__no_fine_grained(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw xor ptr addrspace(1) [[IN]], i32 0 monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw xor ptr addrspace(1) %in, i32 0 monotonic, align 4, !amdgpu.no.fine.grained.memory !0
@@ -252,12 +274,22 @@ define i32 @global_agent_monotonic_idempotent_and(ptr addrspace(1) %in) {
; GFX942-LABEL: global_agent_monotonic_idempotent_and:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v2, -1
+; GFX942-NEXT: global_atomic_and v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_and(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("singlethread") monotonic, align 4
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_and(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*]]:
+; OPT-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; OPT-NEXT: br label %[[ATOMICRMW_START:.*]]
+; OPT: [[ATOMICRMW_START]]:
+; OPT-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[VAL:%.*]], %[[ATOMICRMW_START]] ]
+; OPT-NEXT: [[TMP1:%.*]] = cmpxchg ptr addrspace(1) [[IN]], i32 [[LOADED]], i32 [[LOADED]] syncscope("singlethread") monotonic monotonic, align 4
+; OPT-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; OPT-NEXT: [[VAL]] = extractvalue { i32, i1 } [[TMP1]], 0
+; OPT-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; OPT: [[ATOMICRMW_END]]:
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw and ptr addrspace(1) %in, i32 -1 syncscope("singlethread") monotonic, align 4
@@ -268,16 +300,149 @@ define i32 @global_agent_monotonic_idempotent_and_no_fined_grain(ptr addrspace(1
; GFX942-LABEL: global_agent_monotonic_idempotent_and_no_fined_grain:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v0, v[0:1], off
+; GFX942-NEXT: v_mov_b32_e32 v2, -1
+; GFX942-NEXT: global_atomic_and v0, v[0:1], v2, off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
-; OPT-LABEL: @global_agent_monotonic_idempotent_and_no_fined_grain(
-; OPT-NEXT: entry:
-; OPT-NEXT: [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[IN:%.*]] syncscope("singlethread") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-LABEL: define i32 @global_agent_monotonic_idempotent_and_no_fined_grain(
+; OPT-SAME: ptr addrspace(1) [[IN:%.*]]) {
+; OPT-NEXT: [[ENTRY:.*:]]
+; OPT-NEXT: [[VAL:%.*]] = atomicrmw and ptr addrspace(1) [[IN]], i32 -1 syncscope("singlethread") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; OPT-NEXT: ret i32 [[VAL]]
entry:
%val = atomicrmw and ptr addrspace(1) %in, i32 -1 syncscope("singlethread") monotonic, align 4, !amdgpu.no.fine.grained.memory !0
ret i32 %val
}
+; One thread of a store-buffering litmus t...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218377
More information about the llvm-commits
mailing list