[llvm-branch-commits] [llvm] [AMDGPU] Preserve noclobber after acquire operations from a different addrspace (PR #231855)
Fabian Ritter via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 15 05:53:01 PDT 2026
https://github.com/ritter-x2a updated https://github.com/llvm/llvm-project/pull/231855
>From a9e852d6adfdddb567c2d3b8eebc6e93dcec3f88 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 14 Oct 2026 06:40:49 -0400
Subject: [PATCH] [AMDGPU] Preserve noclobber after acquire operations from a
different addrspace
Follow-up to #219991 to address reviewer comments:
AMDGPUAnnotateUniformValues should annotate uniform global loads as
`!amdgpu.noclobber` if the memory they are reading is never clobbered (since
kernel launch) before the load is executed.
PR #219991 strengthened the logic to consider acquire operations before the
load as clobbering (because they cause stores from other threads to be observed
by the load without a data race).
This PR weakens this criterion to ignore acquire operations that don't affect
the global address space (because they don't affect what the load in question
may or may not see). This addresses the performance-relevant case of s_barriers
with LDS-only fences.
The logic is still overly pessimistic for one-as load acquires, they could also
be ignored when looking for clobbering operations.
Disclaimer: Claude gave review comments and suggestions that I addressed during
development.
For ROCM-29879.
Assisted-by: AI
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 14 +++++++--
llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll | 29 ++++++++++---------
2 files changed, 28 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index c732c42a2cac7..61715ee6b2891 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -411,8 +411,17 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
return !AA->isNoAlias(I->getPointerOperand(), Ptr);
};
- if (const auto *F = dyn_cast<FenceInst>(DefInst))
- return isAcquireOrStronger(F->getOrdering());
+ if (const auto *F = dyn_cast<FenceInst>(DefInst)) {
+ if (!isAcquireOrStronger(F->getOrdering()))
+ return false;
+ SIAtomicAddrSpace AccessedAS =
+ toSIAtomicAddrSpace(Ptr->getType()->getPointerAddressSpace());
+ std::optional<SIAtomicAddrSpace> SynchronizedAS =
+ getSynchronizeAddrSpace(*F);
+ // If SynchronizedAS is std::nullopt, it refers to all address spaces.
+ return !SynchronizedAS ||
+ (*SynchronizedAS & AccessedAS) != SIAtomicAddrSpace::NONE;
+ }
if (const auto *I = dyn_cast<AtomicRMWInst>(DefInst))
return isAcquireOrStronger(I->getOrdering()) || MayAlias(I);
@@ -420,6 +429,7 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
if (const auto *I = dyn_cast<AtomicCmpXchgInst>(DefInst))
return isAcquireOrStronger(I->getMergedOrdering()) || MayAlias(I);
+ // TODO This is pessimistic for one-as load-acquires from a different AS.
if (const auto *I = dyn_cast<LoadInst>(DefInst))
return isAcquireOrStronger(I->getOrdering());
diff --git a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
index 9edb328ee609c..0ecc36b4b9825 100644
--- a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
@@ -1379,7 +1379,7 @@ define amdgpu_kernel void @fence_acquire_one_as_different(ptr addrspace(1) noali
; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(3) [[P3:%.*]]) {
; CHECK-NEXT: [[ACQ:%.*]] = load atomic i32, ptr addrspace(3) [[P3]] monotonic, align 4
; CHECK-NEXT: fence syncscope("workgroup-one-as") acquire, !mmra [[META2:![0-9]+]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4, !amdgpu.noclobber [[META0]]
; CHECK-NEXT: store atomic i32 [[LD]], ptr addrspace(1) [[OUT]] seq_cst, align 4
; CHECK-NEXT: ret void
;
@@ -1392,8 +1392,9 @@ define amdgpu_kernel void @fence_acquire_one_as_different(ptr addrspace(1) noali
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: global_load_dword v1, v0, s[2:3]
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_load_dword s2, s[2:3], 0x0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: global_store_dword v0, v1, s[0:1]
; GCN-NEXT: s_endpgm
%acq = load atomic i32, ptr addrspace(3) %p3 monotonic, align 4
@@ -1408,7 +1409,7 @@ define amdgpu_kernel void @fence_acquire_mmra_different(ptr addrspace(1) noalias
; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(3) [[P3:%.*]]) {
; CHECK-NEXT: [[ACQ:%.*]] = load atomic i32, ptr addrspace(3) [[P3]] monotonic, align 4
; CHECK-NEXT: fence syncscope("workgroup") acquire, !mmra [[META2]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4, !amdgpu.noclobber [[META0]]
; CHECK-NEXT: store atomic i32 [[LD]], ptr addrspace(1) [[OUT]] seq_cst, align 4
; CHECK-NEXT: ret void
;
@@ -1416,13 +1417,14 @@ define amdgpu_kernel void @fence_acquire_mmra_different(ptr addrspace(1) noalias
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dword s6, s[4:5], 0x34
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: ds_read_b32 v1, v1
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: ds_read_b32 v0, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dword v1, v0, s[2:3]
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_load_dword s2, s[2:3], 0x0
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: global_store_dword v0, v1, s[0:1]
; GCN-NEXT: s_endpgm
%acq = load atomic i32, ptr addrspace(3) %p3 monotonic, align 4
@@ -1438,18 +1440,19 @@ define amdgpu_kernel void @local_barrier(ptr addrspace(1) noalias %out, ptr addr
; CHECK-NEXT: fence syncscope("workgroup") release, !mmra [[META2]]
; CHECK-NEXT: tail call void @llvm.amdgcn.s.barrier()
; CHECK-NEXT: fence syncscope("workgroup") acquire, !mmra [[META2]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4, !amdgpu.noclobber [[META0]]
; CHECK-NEXT: store atomic i32 [[LD]], ptr addrspace(1) [[OUT]] seq_cst, align 4
; CHECK-NEXT: ret void
;
; GCN-LABEL: local_barrier:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_barrier
-; GCN-NEXT: global_load_dword v1, v0, s[2:3]
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_load_dword s2, s[2:3], 0x0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: global_store_dword v0, v1, s[0:1]
; GCN-NEXT: s_endpgm
fence syncscope("workgroup") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
More information about the llvm-branch-commits
mailing list