[llvm] [AA] Make getSyncEffects sync-scope aware for pointer arguments (PR #211486)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 00:55:23 PDT 2026
https://github.com/michaelselehov created https://github.com/llvm/llvm-project/pull/211486
getSyncEffects exempted a synchronizing op (fence / stronger-than-monotonic atomic) from clobbering a location whose underlying object never escapes the function. That is unsound for a non-byval pointer argument under a cross-thread sync scope: the pointee is caller-owned and peer threads may hold its address (e.g. a workgroup-shared buffer passed as ptr noalias). noalias/nocapture is a single-thread aliasing property; cross-thread ordering is carried by the sync scope. Under a non-single-thread scope, drop the exemption for every non-byval pointer argument. This is target-independent and only ever widens the result, so it cannot miscompile.
Add an AA hook, getModRefInfoForSyncOp(Loc, AAQI), on the AA Concept/Model with a ModRef default in AAResultBase, to restore precision per target. AMDGPU overrides it to return NoModRef when every underlying object of the location is in private (scratch, addrspace(5)) memory, which is per-workitem and cannot be reached by peer threads under any scope. Fences and all stronger-than-monotonic atomics route through AAResults::getModRefInfoForSyncOp, which chains the registered AAs and intersects with the generic result.
Assisted-by: Claude Opus
>From e58d2f40b49a75aea2f587b75be8e309ac69eef4 Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Thu, 23 Jul 2026 02:14:01 -0500
Subject: [PATCH] [AA] Make getSyncEffects sync-scope aware for pointer
arguments
getSyncEffects exempted a synchronizing op (fence / stronger-than-monotonic
atomic) from clobbering a location whose underlying object never escapes the
function. That is unsound for a non-byval pointer argument under a cross-thread
sync scope: the pointee is caller-owned and peer threads may hold its address
(e.g. a workgroup-shared buffer passed as ptr noalias). noalias/nocapture is a
single-thread aliasing property; cross-thread ordering is carried by the sync
scope. Under a non-single-thread scope, drop the exemption for every non-byval
pointer argument. This is target-independent and only ever widens the result,
so it cannot miscompile.
Add an AA hook, getModRefInfoForSyncOp(Loc, AAQI), on the AA Concept/Model with
a ModRef default in AAResultBase, to restore precision per target. AMDGPU
overrides it to return NoModRef when every underlying object of the location is
in private (scratch, addrspace(5)) memory, which is per-workitem and cannot be
reached by peer threads under any scope. Fences and all stronger-than-monotonic
atomics route through AAResults::getModRefInfoForSyncOp, which chains the
registered AAs and intersects with the generic result.
Assisted-by: Claude Opus
---
llvm/include/llvm/Analysis/AliasAnalysis.h | 29 ++-
llvm/lib/Analysis/AliasAnalysis.cpp | 39 +++-
llvm/lib/Analysis/BasicAliasAnalysis.cpp | 4 +-
.../lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp | 14 ++
llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h | 2 +
llvm/test/Analysis/BasicAA/atomics.ll | 13 +-
.../CodeGen/AMDGPU/gvn-fence-sync-private.ll | 43 ++++
.../CodeGen/AMDGPU/rewrite-out-arguments-2.ll | 9 +-
.../CodeGen/AMDGPU/sync-noalias-private-aa.ll | 104 ++++++++++
.../Transforms/GVN/fence-noalias-syncscope.ll | 185 ++++++++++++++++++
10 files changed, 424 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/gvn-fence-sync-private.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll
create mode 100644 llvm/test/Transforms/GVN/fence-noalias-syncscope.ll
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 4fbe8ea7b8de6..e41eadaa4b457 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -42,6 +42,7 @@
#include "llvm/Analysis/CaptureTracking.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Pass.h"
#include "llvm/Support/Compiler.h"
@@ -606,6 +607,9 @@ class AAResults {
LLVM_ABI ModRefInfo getModRefInfo(const FenceInst *S,
const MemoryLocation &Loc,
AAQueryInfo &AAQI);
+ LLVM_ABI ModRefInfo getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI,
+ SyncScope::ID SSID);
LLVM_ABI ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX,
const MemoryLocation &Loc,
AAQueryInfo &AAQI);
@@ -813,6 +817,12 @@ class LLVM_ABI AAResults::Concept {
const MemoryLocation &Loc,
AAQueryInfo &AAQI) = 0;
+ /// Refine the mod/ref of a synchronizing operation on \p Loc, which it does
+ /// not access directly. Lets a target return NoModRef for address spaces that
+ /// peer threads cannot reach.
+ virtual ModRefInfo getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI) = 0;
+
/// @}
};
@@ -870,6 +880,11 @@ template <typename AAResultT> class AAResults::Model final : public Concept {
AAQueryInfo &AAQI) override {
return Result.getModRefInfo(F, Loc, AAQI);
}
+
+ ModRefInfo getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI) override {
+ return Result.getModRefInfoForSyncOp(Loc, AAQI);
+ }
};
/// A base class to help implement the function alias analysis results concept.
@@ -933,6 +948,11 @@ class AAResultBase {
AAQueryInfo &AAQI) {
return ModRefInfo::ModRef;
}
+
+ ModRefInfo getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI) {
+ return ModRefInfo::ModRef;
+ }
};
/// Return true if this pointer is returned by a noalias function.
@@ -988,9 +1008,14 @@ LLVM_ABI bool isWritableObject(const Value *Object,
bool &ExplicitlyDereferenceableOnly);
/// Get ModRefInfo for a synchronizing operation, such as a fence or stronger
-/// than monotonic atomic load/store.
+/// than monotonic atomic load/store. \p SSID is the operation's sync scope: a
+/// cross-thread scope can order accesses that peer threads perform to
+/// caller-owned memory, so the never-escaping exemption is not applied to a
+/// non-byval pointer argument there. This is target-independent; a target may
+/// restore precision for thread-private address spaces via
+/// AAResults::getModRefInfoForSyncOp.
LLVM_ABI ModRefInfo getSyncEffects(AAResults *AA, const MemoryLocation &Loc,
- AAQueryInfo &AAQI);
+ AAQueryInfo &AAQI, SyncScope::ID SSID);
/// A manager for alias analyses.
///
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index a04bab051f218..2009aacaf62e4 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -459,10 +459,23 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, AliasResult AR) {
//===----------------------------------------------------------------------===//
ModRefInfo llvm::getSyncEffects(AAResults *AA, const MemoryLocation &Loc,
- AAQueryInfo &AAQI) {
+ AAQueryInfo &AAQI, SyncScope::ID SSID) {
if (!Loc.Ptr)
return ModRefInfo::ModRef;
+ // A non-byval pointer argument may alias memory that peer threads access, so
+ // a cross-thread synchronizing operation can order those accesses even if the
+ // pointee does not escape. Such an argument is not exempt; a target can
+ // restore the exemption for thread-private spaces via getModRefInfoForSyncOp.
+ if (SSID != SyncScope::SingleThread) {
+ SmallVector<const Value *, 4> Objects;
+ getUnderlyingObjects(Loc.Ptr, Objects);
+ for (const Value *Object : Objects)
+ if (const auto *Arg = dyn_cast<Argument>(Object))
+ if (!Arg->hasByValAttr())
+ return AA->getModRefInfoMask(Loc);
+ }
+
// If the location is *never* captured, it cannot be affected by
// synchronizing operations. However, we cannot ignore locations that are
// only captured after the operation, as the synchronization may still have
@@ -484,6 +497,20 @@ ModRefInfo llvm::getSyncEffects(AAResults *AA, const MemoryLocation &Loc,
return MR & AA->getModRefInfoMask(Loc);
}
+ModRefInfo AAResults::getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI,
+ SyncScope::ID SSID) {
+ // Let each alias analysis refine the sync effect (e.g. a target that knows an
+ // address space is thread-private), then intersect with the generic result.
+ ModRefInfo Result = ModRefInfo::ModRef;
+ for (const auto &AA : AAs) {
+ Result &= AA->getModRefInfoForSyncOp(Loc, AAQI);
+ if (isNoModRef(Result))
+ return ModRefInfo::NoModRef;
+ }
+ return Result & getSyncEffects(this, Loc, AAQI, SSID);
+}
+
ModRefInfo AAResults::getModRefInfo(const LoadInst *L,
const MemoryLocation &Loc,
AAQueryInfo &AAQI) {
@@ -494,7 +521,7 @@ ModRefInfo AAResults::getModRefInfo(const LoadInst *L,
if (AR == AliasResult::NoAlias) {
// Synchronization effects may affect locations that do not alias.
if (isStrongerThanMonotonic(L->getOrdering()))
- return getSyncEffects(this, Loc, AAQI);
+ return getModRefInfoForSyncOp(Loc, AAQI, L->getSyncScopeID());
return ModRefInfo::NoModRef;
}
}
@@ -517,7 +544,7 @@ ModRefInfo AAResults::getModRefInfo(const StoreInst *S,
if (AR == AliasResult::NoAlias) {
// Synchronization effects may affect locations that do not alias.
if (isStrongerThanMonotonic(S->getOrdering()))
- return getSyncEffects(this, Loc, AAQI);
+ return getModRefInfoForSyncOp(Loc, AAQI, S->getSyncScopeID());
return ModRefInfo::NoModRef;
}
@@ -550,7 +577,7 @@ ModRefInfo AAResults::getModRefInfo(const FenceInst *F,
return ModRefInfo::NoModRef;
}
- return Result & getSyncEffects(this, Loc, AAQI);
+ return Result & getModRefInfoForSyncOp(Loc, AAQI, F->getSyncScopeID());
}
return ModRefInfo::ModRef;
@@ -611,7 +638,7 @@ ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX,
if (AR == AliasResult::NoAlias) {
// Synchronization effects may affect locations that do not alias.
if (isStrongerThanMonotonic(CX->getMergedOrdering()))
- return getSyncEffects(this, Loc, AAQI);
+ return getModRefInfoForSyncOp(Loc, AAQI, CX->getSyncScopeID());
return ModRefInfo::NoModRef;
}
}
@@ -629,7 +656,7 @@ ModRefInfo AAResults::getModRefInfo(const AtomicRMWInst *RMW,
if (AR == AliasResult::NoAlias) {
// Synchronization effects may affect locations that do not alias.
if (isStrongerThanMonotonic(RMW->getOrdering()))
- return getSyncEffects(this, Loc, AAQI);
+ return getModRefInfoForSyncOp(Loc, AAQI, RMW->getSyncScopeID());
return ModRefInfo::NoModRef;
}
}
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 7df62577e04db..d17b2fd2067f3 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -972,7 +972,9 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
ModRefInfo SyncMR = ModRefInfo::NoModRef;
if (isModAndRefSet(OtherMR) && Call->maySynchronize() &&
Call->isInlineAsm()) {
- SyncMR = getSyncEffects(&AAQI.AAR, Loc, AAQI);
+ // A call has no explicit sync scope; assume the widest (cross-thread)
+ // scope so the exemption is not applied to caller-provided arguments.
+ SyncMR = getSyncEffects(&AAQI.AAR, Loc, AAQI, SyncScope::System);
if (isModAndRefSet(SyncMR))
return SyncMR;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
index 7bcc128cb114f..3cdd820ed11fd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
@@ -114,3 +114,17 @@ ModRefInfo AMDGPUAAResult::getModRefInfoMask(const MemoryLocation &Loc,
return ModRefInfo::ModRef;
}
+
+ModRefInfo AMDGPUAAResult::getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI) {
+ // Private (scratch) memory is per-workitem and unreachable by peer threads.
+ // Check every underlying object so an addrspacecast is not read as private.
+ if (!Loc.Ptr)
+ return ModRefInfo::ModRef;
+ SmallVector<const Value *, 4> Objects;
+ getUnderlyingObjects(Loc.Ptr, Objects);
+ for (const Value *Obj : Objects)
+ if (Obj->getType()->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS)
+ return ModRefInfo::ModRef;
+ return ModRefInfo::NoModRef;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
index aa0e9974667b7..8f2b2bf87b2e0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
@@ -41,6 +41,8 @@ class AMDGPUAAResult : public AAResultBase {
AAQueryInfo &AAQI, const Instruction *CtxI);
ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI,
bool IgnoreLocals);
+ ModRefInfo getModRefInfoForSyncOp(const MemoryLocation &Loc,
+ AAQueryInfo &AAQI);
};
/// Analysis pass providing a never-invalidated alias analysis result.
diff --git a/llvm/test/Analysis/BasicAA/atomics.ll b/llvm/test/Analysis/BasicAA/atomics.ll
index b3001be0f4063..c92ad29e8988f 100644
--- a/llvm/test/Analysis/BasicAA/atomics.ll
+++ b/llvm/test/Analysis/BasicAA/atomics.ll
@@ -139,16 +139,19 @@ define void @alloca_no_escape_aliasing() {
ret void
}
+; A cross-thread sync op may be ordered against peer-thread accesses to a
+; caller-owned pointer argument, so %a is ModRef here even though it does not
+; escape this function.
; CHECK-LABEL: Function: noalias_no_escape:
-; CHECK: NoModRef: Ptr: i32* %a <-> fence release
+; CHECK: Both ModRef: Ptr: i32* %a <-> fence release
; CHECK: Both ModRef: Ptr: i32* %x <-> fence release
-; CHECK: NoModRef: Ptr: i32* %a <-> %1 = atomicrmw add ptr %x, i32 1 acq_rel, align 4
+; CHECK: Both ModRef: Ptr: i32* %a <-> %1 = atomicrmw add ptr %x, i32 1 acq_rel, align 4
; CHECK: Both ModRef: Ptr: i32* %x <-> %1 = atomicrmw add ptr %x, i32 1 acq_rel, align 4
-; CHECK: NoModRef: Ptr: i32* %a <-> %2 = cmpxchg ptr %x, i32 0, i32 1 acq_rel monotonic, align 4
+; CHECK: Both ModRef: Ptr: i32* %a <-> %2 = cmpxchg ptr %x, i32 0, i32 1 acq_rel monotonic, align 4
; CHECK: Both ModRef: Ptr: i32* %x <-> %2 = cmpxchg ptr %x, i32 0, i32 1 acq_rel monotonic, align 4
-; CHECK: NoModRef: Ptr: i32* %a <-> %3 = load atomic i32, ptr %x acquire, align 4
+; CHECK: Both ModRef: Ptr: i32* %a <-> %3 = load atomic i32, ptr %x acquire, align 4
; CHECK: Both ModRef: Ptr: i32* %x <-> %3 = load atomic i32, ptr %x acquire, align 4
-; CHECK: NoModRef: Ptr: i32* %a <-> store atomic i32 0, ptr %x release, align 4
+; CHECK: Both ModRef: Ptr: i32* %a <-> store atomic i32 0, ptr %x release, align 4
; CHECK: Both ModRef: Ptr: i32* %x <-> store atomic i32 0, ptr %x release, align 4
define void @noalias_no_escape(ptr noalias %a, ptr %x) {
store i32 0, ptr %a
diff --git a/llvm/test/CodeGen/AMDGPU/gvn-fence-sync-private.ll b/llvm/test/CodeGen/AMDGPU/gvn-fence-sync-private.ll
new file mode 100644
index 0000000000000..92cf5863387ba
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/gvn-fence-sync-private.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=gvn -aa-pipeline=amdgpu-aa,basic-aa -S < %s | FileCheck %s
+
+; End-to-end check that a cross-thread fence is honored per address space on
+; AMDGPU. See llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll for the AA
+; result this relies on, and llvm/test/Transforms/GVN/fence-noalias-syncscope.ll
+; for the target-independent behavior.
+
+; Workgroup-shared (LDS/local, AS3) memory is reachable by peer threads, so the
+; fence may order their writes: the reload must be preserved (two loads remain).
+define i32 @fence_keeps_local(ptr addrspace(3) noalias %p) {
+; CHECK-LABEL: define i32 @fence_keeps_local(
+; CHECK-SAME: ptr addrspace(3) noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(3) [[P]], align 4
+; CHECK-NEXT: fence syncscope("agent") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr addrspace(3) [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr addrspace(3) %p
+ fence syncscope("agent") acq_rel
+ %v2 = load i32, ptr addrspace(3) %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Private (scratch, AS5) memory is per-workitem and unreachable by peers, so the
+; fence cannot order peer writes to it: the second load is forwarded to the
+; first (only one load remains).
+define i32 @fence_forwards_private(ptr addrspace(5) noalias %p) {
+; CHECK-LABEL: define i32 @fence_forwards_private(
+; CHECK-SAME: ptr addrspace(5) noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: fence syncscope("agent") acq_rel
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V1]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr addrspace(5) %p
+ fence syncscope("agent") acq_rel
+ %v2 = load i32, ptr addrspace(5) %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
index f836bd61ff864..5a9724758707e 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-rewrite-out-arguments < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-aa-wrapper -amdgpu-aa -amdgpu-rewrite-out-arguments < %s | FileCheck %s
define i32 @load_out_ptr_after_store(ptr addrspace(5) %out) {
; CHECK-LABEL: define i32 @load_out_ptr_after_store(
@@ -315,9 +315,10 @@ define void @fence_release_after_store(ptr addrspace(5) %out) {
define void @fence_seq_cst_after_store(ptr addrspace(5) %out) {
; CHECK-LABEL: define void @fence_seq_cst_after_store(
-; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
-; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
-; CHECK-NEXT: fence seq_cst
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP2:%.*]] = call [[FENCE_SEQ_CST_AFTER_STORE:%.*]] @[[FENCE_SEQ_CST_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue [[FENCE_SEQ_CST_AFTER_STORE]] [[TMP2]], 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
; CHECK-NEXT: ret void
;
store i32 0, ptr addrspace(5) %out
diff --git a/llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll b/llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll
new file mode 100644
index 0000000000000..a65ae34613c16
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll
@@ -0,0 +1,104 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=aa-eval -aa-pipeline=amdgpu-aa,basic-aa -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+
+; Generic AA conservatively reports ModRef for a synchronizing operation against
+; a non-byval pointer argument (see llvm/test/Transforms/GVN/fence-noalias-syncscope.ll).
+; AMDGPU AA restores precision for private (scratch) memory, which is per-workitem
+; and cannot be reached by peer threads, while keeping LDS/local and global memory
+; conservative because those are shared.
+
+; Private (scratch) argument: a cross-thread sync op is NoModRef.
+; CHECK-LABEL: Function: private:
+; CHECK: NoModRef: Ptr: i32* %a <-> fence syncscope("agent") acq_rel
+; CHECK: NoModRef: Ptr: i32* %a <-> {{.*}}atomicrmw add ptr addrspace(5) %x
+; CHECK: NoModRef: Ptr: i32* %a <-> {{.*}}load atomic i32, ptr addrspace(5) %x
+; CHECK: NoModRef: Ptr: i32* %a <-> store atomic i32 0, ptr addrspace(5) %x release
+define void @private(ptr addrspace(5) noalias %a, ptr addrspace(5) noalias %x) {
+ store i32 0, ptr addrspace(5) %a
+ fence syncscope("agent") acq_rel
+ atomicrmw add ptr addrspace(5) %x, i32 1 acq_rel
+ load atomic i32, ptr addrspace(5) %x acquire, align 4
+ store atomic i32 0, ptr addrspace(5) %x release, align 4
+ ret void
+}
+
+; LDS/local is workgroup-shared: a cross-thread sync op is ModRef.
+; CHECK-LABEL: Function: local:
+; CHECK: Both ModRef: Ptr: i32* %a <-> fence syncscope("agent") acq_rel
+; CHECK: Both ModRef: Ptr: i32* %a <-> {{.*}}atomicrmw add ptr addrspace(3) %x
+define void @local(ptr addrspace(3) noalias %a, ptr addrspace(3) noalias %x) {
+ store i32 0, ptr addrspace(3) %a
+ fence syncscope("agent") acq_rel
+ atomicrmw add ptr addrspace(3) %x, i32 1 acq_rel
+ ret void
+}
+
+; Global memory is shared: a cross-thread sync op is ModRef.
+; CHECK-LABEL: Function: global:
+; CHECK: Both ModRef: Ptr: i32* %a <-> fence syncscope("agent") acq_rel
+; CHECK: Both ModRef: Ptr: i32* %a <-> {{.*}}atomicrmw add ptr addrspace(1) %x
+define void @global(ptr addrspace(1) noalias %a, ptr addrspace(1) noalias %x) {
+ store i32 0, ptr addrspace(1) %a
+ fence syncscope("agent") acq_rel
+ atomicrmw add ptr addrspace(1) %x, i32 1 acq_rel
+ ret void
+}
+
+; A private pointer obtained by addrspacecast from generic is not proof of
+; peer-unreachability (the underlying object is generic), so it stays ModRef.
+; CHECK-LABEL: Function: cast_generic_to_private:
+; CHECK: Both ModRef: Ptr: i32* %p <-> fence syncscope("agent") acq_rel
+; CHECK: Both ModRef: Ptr: i32* %p <-> {{.*}}atomicrmw add ptr addrspace(5) %x
+define void @cast_generic_to_private(ptr noalias %g, ptr addrspace(5) noalias %x) {
+ %p = addrspacecast ptr %g to ptr addrspace(5)
+ store i32 0, ptr addrspace(5) %p
+ fence syncscope("agent") acq_rel
+ atomicrmw add ptr addrspace(5) %x, i32 1 acq_rel
+ ret void
+}
+
+; select of two private pointers: every underlying object is private, NoModRef.
+; CHECK-LABEL: Function: select_private:
+; CHECK: NoModRef: Ptr: i32* %sel <-> fence syncscope("agent") acq_rel
+define void @select_private(ptr addrspace(5) noalias %a, ptr addrspace(5) noalias %b, i1 %c) {
+ %sel = select i1 %c, ptr addrspace(5) %a, ptr addrspace(5) %b
+ store i32 0, ptr addrspace(5) %sel
+ fence syncscope("agent") acq_rel
+ ret void
+}
+
+; phi of two private pointers: every underlying object is private, NoModRef.
+; CHECK-LABEL: Function: phi_private:
+; CHECK: NoModRef: Ptr: i32* %p <-> fence syncscope("agent") acq_rel
+define void @phi_private(ptr addrspace(5) noalias %a, ptr addrspace(5) noalias %b, i1 %c) {
+entry:
+ br i1 %c, label %t, label %f
+t:
+ br label %j
+f:
+ br label %j
+j:
+ %p = phi ptr addrspace(5) [ %a, %t ], [ %b, %f ]
+ store i32 0, ptr addrspace(5) %p
+ fence syncscope("agent") acq_rel
+ ret void
+}
+
+; A pointer loaded from memory is still typed private (AS5 is always scratch),
+; so it remains peer-unreachable: NoModRef.
+; CHECK-LABEL: Function: loaded_private_ptr:
+; CHECK: NoModRef: Ptr: i32* %p <-> fence syncscope("agent") acq_rel
+define void @loaded_private_ptr(ptr addrspace(5) noalias %pp) {
+ %p = load ptr addrspace(5), ptr addrspace(5) %pp
+ store i32 0, ptr addrspace(5) %p
+ fence syncscope("agent") acq_rel
+ ret void
+}
+
+; byval private argument is a per-lane copy in scratch: NoModRef.
+; CHECK-LABEL: Function: byval_private:
+; CHECK: NoModRef: Ptr: i32* %a <-> fence syncscope("agent") acq_rel
+define void @byval_private(ptr addrspace(5) byval(i32) noalias %a) {
+ store i32 0, ptr addrspace(5) %a
+ fence syncscope("agent") acq_rel
+ ret void
+}
diff --git a/llvm/test/Transforms/GVN/fence-noalias-syncscope.ll b/llvm/test/Transforms/GVN/fence-noalias-syncscope.ll
new file mode 100644
index 0000000000000..c42e7c40384cd
--- /dev/null
+++ b/llvm/test/Transforms/GVN/fence-noalias-syncscope.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=gvn -S < %s | FileCheck %s
+
+; A cross-thread fence orders accesses to caller-owned memory, so a load of a
+; non-byval pointer argument cannot be forwarded across it, regardless of its
+; address space (generic AA is target-independent; a target AA may restore
+; precision for thread-private address spaces, see
+; llvm/test/CodeGen/AMDGPU/sync-noalias-private-aa.ll). The exemption still
+; applies to thread-private objects (alloca, byval) and to a single-thread
+; scope fence. (`noalias` is only used so GVN would otherwise forward the
+; reload; a may-alias argument behaves the same.)
+
+; Cross-thread scope, non-byval pointer argument: the reload must be kept.
+define i32 @noalias_arg_workgroup(ptr noalias %p) {
+; CHECK-LABEL: define i32 @noalias_arg_workgroup(
+; CHECK-SAME: ptr noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr %p
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Same, with the pointer in another cross-thread-visible address space: the
+; decision is identical, i.e. it does not depend on the specific address space.
+define i32 @noalias_arg_workgroup_as3(ptr addrspace(3) noalias %p) {
+; CHECK-LABEL: define i32 @noalias_arg_workgroup_as3(
+; CHECK-SAME: ptr addrspace(3) noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(3) [[P]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr addrspace(3) [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr addrspace(3) %p
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr addrspace(3) %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; A non-zero address space is not special to generic AA: without a target AA
+; that models it as thread-private, the reload is still kept.
+define i32 @noalias_arg_private(ptr addrspace(5) noalias %p) {
+; CHECK-LABEL: define i32 @noalias_arg_private(
+; CHECK-SAME: ptr addrspace(5) noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr addrspace(5) %p
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr addrspace(5) %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Single-thread scope (e.g. atomic_signal_fence): the exemption is valid, so the
+; reload is forwarded.
+define i32 @noalias_arg_singlethread(ptr noalias %p) {
+; CHECK-LABEL: define i32 @noalias_arg_singlethread(
+; CHECK-SAME: ptr noalias [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: fence syncscope("singlethread") acq_rel
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V1]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr %p
+ fence syncscope("singlethread") acq_rel
+ %v2 = load i32, ptr %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; A byval argument is a private per-call copy: the reload is forwarded.
+define i32 @byval_arg_workgroup(ptr byval(i32) %p) {
+; CHECK-LABEL: define i32 @byval_arg_workgroup(
+; CHECK-SAME: ptr byval(i32) [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V1]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr %p
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; A local alloca is thread-private: the reload is forwarded.
+define i32 @alloca_workgroup() {
+; CHECK-LABEL: define i32 @alloca_workgroup() {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: store i32 42, ptr addrspace(5) [[A]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: ret i32 84
+;
+ %a = alloca i32, addrspace(5)
+ store i32 42, ptr addrspace(5) %a
+ %v1 = load i32, ptr addrspace(5) %a
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr addrspace(5) %a
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; A select between two caller-owned arguments: getUnderlyingObjects sees both
+; arguments (not just a single object), so the reload is kept across the fence.
+define i32 @select_noalias_args(ptr noalias %p, ptr noalias %q, i1 %c) {
+; CHECK-LABEL: define i32 @select_noalias_args(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr [[P]], ptr [[Q]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SEL]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SEL]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %sel = select i1 %c, ptr %p, ptr %q
+ %v1 = load i32, ptr %sel
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr %sel
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Same via a phi of two caller-owned arguments: the reload must be kept.
+define i32 @phi_noalias_args(ptr noalias %p, ptr noalias %q, i1 %c) {
+; CHECK-LABEL: define i32 @phi_noalias_args(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[TA:.*]], label %[[TB:.*]]
+; CHECK: [[TA]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[TB]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[PTR:%.*]] = phi ptr [ [[P]], %[[TA]] ], [ [[Q]], %[[TB]] ]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+entry:
+ br i1 %c, label %ta, label %tb
+ta:
+ br label %join
+tb:
+ br label %join
+join:
+ %ptr = phi ptr [ %p, %ta ], [ %q, %tb ]
+ %v1 = load i32, ptr %ptr
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr %ptr
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Control: no `noalias`, so the never-escaping exemption never fires and the
+; reload is kept regardless.
+define i32 @mayalias_arg_workgroup(ptr %p) {
+; CHECK-LABEL: define i32 @mayalias_arg_workgroup(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: fence syncscope("workgroup") acq_rel
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %v1 = load i32, ptr %p
+ fence syncscope("workgroup") acq_rel
+ %v2 = load i32, ptr %p
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
More information about the llvm-commits
mailing list