[llvm] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer fat pointer atomics (PR #213585)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 2 21:05:28 PDT 2026
https://github.com/aobolensk created https://github.com/llvm/llvm-project/pull/213585
Buffer atomic/load/store intrinsics have no scope operand, so the scope was lost during lowering
Stash it as metadata and recover it in SIISelLowering so SIMemoryLegalizer sets cache bypass bits
>From 1d254511c60af3c1cb7575a5caa83635412a4915 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 31 Jul 2026 17:18:47 +0200
Subject: [PATCH] [AMDGPU] Fix missing cache bypass scope on seq_cst buffer fat
pointer atomics
Buffer atomic/load/store intrinsics have no scope operand, so the
scope was lost during lowering
Stash it as metadata and recover it in SIISelLowering so SIMemoryLegalizer sets cache bypass bits
---
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 16 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 27 +++
.../AMDGPU/memory-legalizer-buffer-atomics.ll | 155 +++++++++++++++++-
3 files changed, 196 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index dd2f7075c8df2..bb12289821749 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1566,6 +1566,10 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx);
void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID);
+ // Buffer intrinsics have no ordering/scope operand, so stash it as
+ // metadata for SIISelLowering::getTgtMemIntrinsic to recover.
+ void attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID);
Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty,
Align Alignment, AtomicOrdering Order,
bool IsVolatile, SyncScope::ID SSID);
@@ -1872,6 +1876,16 @@ void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) {
Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A));
}
+void SplitPtrStructs::attachAtomicScopeMD(CallInst *Call, AtomicOrdering Order,
+ SyncScope::ID SSID) {
+ if (Order == AtomicOrdering::NotAtomic)
+ return;
+ LLVMContext &Ctx = Call->getContext();
+ StringRef Scope = Ctx.getSyncScopeName(SSID).value_or("");
+ Call->setMetadata("amdgpu.buffer.atomic.scope",
+ MDNode::get(Ctx, MDString::get(Ctx, Scope)));
+}
+
void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order,
SyncScope::ID SSID) {
switch (Order) {
@@ -2023,6 +2037,7 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
CallInst *Call = IRB.CreateIntrinsicWithoutFolding(IID, Ty, Args);
copyMetadata(Call, I);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, Alignment, Arg ? 1 : 0);
Call->takeName(I);
@@ -2093,6 +2108,7 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, Off,
IRB.getInt32(0), IRB.getInt32(Aux)});
copyMetadata(Call, &AI);
+ attachAtomicScopeMD(Call, Order, SSID);
setAlign(Call, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 403f5d7476dfc..dc9725a7c2de9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1416,6 +1416,16 @@ static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
return CI.getContext().getOrInsertSyncScopeID(Scope);
}
+// Recovers the scope AMDGPULowerBufferFatPointers stashed as metadata,
+// since buffer intrinsics have no scope operand of their own.
+static std::optional<SyncScope::ID> parseAtomicScopeMD(const CallBase &CI) {
+ MDNode *ScopeMD = CI.getMetadata("amdgpu.buffer.atomic.scope");
+ if (!ScopeMD)
+ return std::nullopt;
+ StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
+ return CI.getContext().getOrInsertSyncScopeID(Scope);
+}
+
void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
const CallBase &CI,
MachineFunction &MF,
@@ -1503,6 +1513,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = getValueType(MF.getDataLayout(), DataTy);
Info.flags = Flags | MachineMemOperand::MOStore;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so SIMemoryLegalizer::expandStore sets cache
+ // bypass bits, since atomic and plain buffer stores look identical.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
} else {
// Atomic, NoReturn Sampler or prefetch
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
@@ -1521,6 +1537,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// XXX - Should this be volatile without known ordering?
Info.flags |= MachineMemOperand::MOVolatile;
Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandAtomicCmpxchgOrRmw sets cache
+ // bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
}
break;
case Intrinsic::amdgcn_raw_buffer_load_lds:
@@ -1561,6 +1583,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags = Flags | MachineMemOperand::MOLoad;
+ if (std::optional<SyncScope::ID> SSID = parseAtomicScopeMD(CI)) {
+ // Pretend to be atomic so expandLoad sets cache bypass bits.
+ Info.order = AtomicOrdering::Monotonic;
+ Info.ssid = *SSID;
+ }
Infos.push_back(Info);
return;
}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
index 1cf8ef7a918f4..6523c168dd50c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-buffer-atomics.ll
@@ -22,7 +22,7 @@ define void @buffer_fat_ptr_agent_atomic_add_noret_i32(ptr addrspace(7) inreg %p
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_DEV
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret void
@@ -49,13 +49,164 @@ define i32 @buffer_fat_ptr_agent_atomic_add_ret_i32(ptr addrspace(7) inreg %ptr,
; GFX1250-NEXT: s_mov_b32 s3, s8
; GFX1250-NEXT: v_mov_b32_e32 v1, s16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = atomicrmw add ptr addrspace(7) %ptr, i32 %val syncscope("agent") monotonic
ret i32 %ret
}
+define void @buffer_fat_ptr_seq_cst_atomic_add_noret_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_atomic_add_noret_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_add_u32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %unused = atomicrmw add ptr addrspace(7) %ptr, i32 %val seq_cst
+ ret void
+}
+
+define i32 @buffer_fat_ptr_seq_cst_cmpxchg_i32(ptr addrspace(7) inreg %ptr, i32 %cmp, i32 %new) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_cmpxchg_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: scratch_store_b32 off, v1, s32 nv ; 4-byte Folded Spill
+; GFX1250-NEXT: v_mov_b32_e32 v2, v0
+; GFX1250-NEXT: scratch_load_b32 v0, off, s32 nv ; 4-byte Folded Reload
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_mov_b32_e32 v1, v2
+; GFX1250-NEXT: v_mov_b32_e32 v2, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %pair = cmpxchg ptr addrspace(7) %ptr, i32 %cmp, i32 %new seq_cst seq_cst
+ %ret = extractvalue { i32, i1 } %pair, 0
+ ret i32 %ret
+}
+
+define i32 @buffer_fat_ptr_seq_cst_load_i32(ptr addrspace(7) inreg %ptr) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_load_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s16
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %ret = load atomic i32, ptr addrspace(7) %ptr seq_cst, align 4
+ ret i32 %ret
+}
+
+define void @buffer_fat_ptr_seq_cst_store_i32(ptr addrspace(7) inreg %ptr, i32 %val) {
+; GFX1250-LABEL: buffer_fat_ptr_seq_cst_store_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s6, s2
+; GFX1250-NEXT: s_mov_b32 s4, s0
+; GFX1250-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
+; GFX1250-NEXT: s_mov_b32 s7, s3
+; GFX1250-NEXT: s_mov_b32 s8, s7
+; GFX1250-NEXT: s_mov_b32 s9, s6
+; GFX1250-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX1250-NEXT: s_mov_b32 s5, s1
+; GFX1250-NEXT: s_mov_b32 s10, s5
+; GFX1250-NEXT: s_mov_b32 s0, s4
+; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; GFX1250-NEXT: s_mov_b32 s1, s10
+; GFX1250-NEXT: s_mov_b32 s2, s9
+; GFX1250-NEXT: s_mov_b32 s3, s8
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ store atomic i32 %val, ptr addrspace(7) %ptr seq_cst, align 4
+ ret void
+}
+
define void @raw_buffer_atomic_add_v2f16_noret(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
; GFX1250-LABEL: raw_buffer_atomic_add_v2f16_noret:
; GFX1250: ; %bb.0:
More information about the llvm-commits
mailing list