[llvm] [AMDGPU] Employ MemorySSA when rewriting out arguments (NFCI) (PR #207922)
Antonio Frighetto via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 7 00:58:30 PDT 2026
https://github.com/antoniofrighetto created https://github.com/llvm/llvm-project/pull/207922
While transitioning away from MemoryDependenceAnalysis, replace MDA `getPointerDependencyFrom` query with a reverse traversal of the memory accesses of the return basic block (MemoryDefs and MemoryUses), looking for the must-aliasing store to the out argument. The existing behaviour is meant to be preserved.
---
Unlike MemorySSA APIs, MDA getPointerDependencyFrom() takes a QueryInst as its last argument, which, among other things, is used when deciding whether an intervening atomic load may be a dependency. In our case, QueryInst is always a return instruction, which doesn’t read or write memory; thus, atomic loads with monotonic ordering are treated as non-clobbering wrt the out argument, per the following logic:
https://github.com/llvm/llvm-project/blob/0075a8fbcc9928ef5a04a30233b08f67219a4310/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp#L492-L499
Besides, may-alias reads are also walked past, since getPointerDependencyFrom() `isLoad` parameter is set. Conversely, MSSA (via AA) is more conservative and says ModRef for any load stronger than unordered (regardless of query context):
https://github.com/llvm/llvm-project/blob/2cbaca8fe5f33069d764c6cada16ea4305a6efc5/llvm/lib/Analysis/AliasAnalysis.cpp#L502-L504
Hence the dedicated handling for atomic loads and fences to preserve the behavior.
The precommitted tests extend coverage for these paths, although I'm unclear whether they occur in practice or map cleanly to AMDGPU semantics (at least, they seem to preserve equivalence for now).
An alternative (simpler) approach would be a backward scan of BB looking for the last store to the out argument:
https://github.com/antoniofrighetto/llvm-project/commit/bfd8550bae7356857ec9983a9f595314d954e5ad.
Left a TODO in the code as I’m also unsure whether the pass is (meant to be) semantics-preserving. In the following case: https://llvm.godbolt.org/z/7fP35jfsx, when both arguments passed to `@callee` alias the same slot, `%out` ends up holding 0 in the original program. After the rewrite, the store into `%val` has been sunk past the load in the stub, thus `%other` observes the pre-call value and `%out` ends up holding 1.
@arsenm Does this make sense?
>From 5b3e83cccf35477110b809da828ec3e7bfe6673f Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me at antoniofrighetto.com>
Date: Tue, 7 Jul 2026 09:28:21 +0200
Subject: [PATCH 1/2] [AMDGPU] Introduce tests (NFC)
---
.../CodeGen/AMDGPU/rewrite-out-arguments-2.ll | 349 ++++++++++++++++++
1 file changed, 349 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
new file mode 100644
index 0000000000000..5a35d603e9703
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments-2.ll
@@ -0,0 +1,349 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-rewrite-out-arguments < %s | FileCheck %s
+
+define i32 @load_out_ptr_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define i32 @load_out_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: ret i32 [[LOAD]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load i32, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define i32 @load_out_ptr_before_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define i32 @load_out_ptr_before_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: ret i32 [[LOAD]]
+;
+ %load = load i32, ptr addrspace(5) %out
+ store i32 0, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define i32 @volatile_load_out_ptr_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define i32 @volatile_load_out_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: [[LOAD:%.*]] = load volatile i32, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: ret i32 [[LOAD]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load volatile i32, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define i32 @atomic_load_out_ptr_before_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define i32 @atomic_load_out_ptr_before_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: [[LOAD:%.*]] = load atomic i32, ptr addrspace(5) [[OUT]] monotonic, align 4
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: ret i32 [[LOAD]]
+;
+ %load = load atomic i32, ptr addrspace(5) %out monotonic, align 4
+ store i32 0, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define void @volatile_store_out_ptr(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @volatile_store_out_ptr(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store volatile i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ store volatile i32 0, ptr addrspace(5) %out
+ ret void
+}
+
+define void @atomic_store_out_ptr(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @atomic_store_out_ptr(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store atomic i32 0, ptr addrspace(5) [[OUT]] monotonic, align 4
+; CHECK-NEXT: ret void
+;
+ store atomic i32 0, ptr addrspace(5) %out monotonic, align 4
+ ret void
+}
+
+define void @atomic_store_out_ptr_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @atomic_store_out_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: store atomic i32 1, ptr addrspace(5) [[OUT]] release, align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ store atomic i32 1, ptr addrspace(5) %out release, align 4
+ ret void
+}
+
+define i32 @load_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @load_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[LOAD_PTR_AFTER_STORE:%.*]] @[[LOAD_PTR_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[LOAD_PTR_AFTER_STORE]] [[TMP3]], 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[LOAD_PTR_AFTER_STORE]] [[TMP3]], 0
+; CHECK-NEXT: ret i32 [[TMP5]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load i32, ptr addrspace(5) %ptr
+ ret i32 %load
+}
+
+define i32 @volatile_load_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @volatile_load_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[VOLATILE_LOAD_PTR_AFTER_STORE:%.*]] @[[VOLATILE_LOAD_PTR_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[VOLATILE_LOAD_PTR_AFTER_STORE]] [[TMP3]], 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[VOLATILE_LOAD_PTR_AFTER_STORE]] [[TMP3]], 0
+; CHECK-NEXT: ret i32 [[TMP5]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load volatile i32, ptr addrspace(5) %ptr
+ ret i32 %load
+}
+
+define i32 @atomic_load_acquire_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @atomic_load_acquire_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: [[LOAD:%.*]] = load atomic i32, ptr addrspace(5) [[PTR]] acquire, align 4
+; CHECK-NEXT: ret i32 [[LOAD]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load atomic i32, ptr addrspace(5) %ptr acquire, align 4
+ ret i32 %load
+}
+
+define i32 @atomic_load_monotonic_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @atomic_load_monotonic_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[ATOMIC_LOAD_MONOTONIC_PTR_AFTER_STORE:%.*]] @[[ATOMIC_LOAD_MONOTONIC_PTR_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[ATOMIC_LOAD_MONOTONIC_PTR_AFTER_STORE]] [[TMP3]], 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[ATOMIC_LOAD_MONOTONIC_PTR_AFTER_STORE]] [[TMP3]], 0
+; CHECK-NEXT: ret i32 [[TMP5]]
+;
+ store i32 0, ptr addrspace(5) %out
+ %load = load atomic i32, ptr addrspace(5) %ptr monotonic, align 4
+ ret i32 %load
+}
+
+define void @volatile_store_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @volatile_store_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: store volatile i32 1, ptr addrspace(5) [[PTR]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ store volatile i32 1, ptr addrspace(5) %ptr
+ ret void
+}
+
+define void @atomic_store_release_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @atomic_store_release_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: store atomic i32 1, ptr addrspace(5) [[PTR]] release, align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ store atomic i32 1, ptr addrspace(5) %ptr release, align 4
+ ret void
+}
+
+define void @atomic_store_unordered_ptr_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @atomic_store_unordered_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: store atomic i32 1, ptr addrspace(5) [[PTR]] unordered, align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ store atomic i32 1, ptr addrspace(5) %ptr unordered, align 4
+ ret void
+}
+
+define void @opaque_read_call_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @opaque_read_call_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP2:%.*]] = call [[OPAQUE_READ_CALL_AFTER_STORE:%.*]] @[[OPAQUE_READ_CALL_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue [[OPAQUE_READ_CALL_AFTER_STORE]] [[TMP2]], 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @opaque_read()
+ ret void
+}
+
+define void @opaque_rw_call_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @opaque_rw_call_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: call void @opaque_rw_argmem(ptr addrspace(5) [[PTR]])
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @opaque_rw_argmem(ptr addrspace(5) %ptr)
+ ret void
+}
+
+define void @opaque_call_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @opaque_call_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: call void @opaque()
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @opaque()
+ ret void
+}
+
+define void @memcpy_after_store(ptr addrspace(5) %out, ptr addrspace(5) %src, ptr addrspace(5) noalias %dst) {
+; CHECK-LABEL: define void @memcpy_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]], ptr addrspace(5) noalias [[TMP2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP4:%.*]] = call [[MEMCPY_AFTER_STORE:%.*]] @[[MEMCPY_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]], ptr addrspace(5) [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[MEMCPY_AFTER_STORE]] [[TMP4]], 0
+; CHECK-NEXT: store i32 [[TMP5]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @llvm.memcpy.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 4, i1 false)
+ ret void
+}
+
+define void @memset_after_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @memset_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]], ptr addrspace(5) [[PTR:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: call void @llvm.memset.p5.i32(ptr addrspace(5) [[PTR]], i8 0, i32 4, i1 false)
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @llvm.memset.p5.i32(ptr addrspace(5) %ptr, i8 0, i32 4, i1 false)
+ ret void
+}
+
+define void @memset_noalias_ptr_after_store(ptr addrspace(5) noalias %out, ptr addrspace(5) noalias %ptr) {
+; CHECK-LABEL: define void @memset_noalias_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) noalias [[TMP0:%.*]], ptr addrspace(5) noalias [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[MEMSET_NOALIAS_PTR_AFTER_STORE:%.*]] @[[MEMSET_NOALIAS_PTR_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[MEMSET_NOALIAS_PTR_AFTER_STORE]] [[TMP3]], 0
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ call void @llvm.memset.p5.i32(ptr addrspace(5) %ptr, i8 0, i32 4, i1 false)
+ ret void
+}
+
+define i32 @load_ptr_before_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @load_ptr_before_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[LOAD_PTR_BEFORE_STORE:%.*]] @[[LOAD_PTR_BEFORE_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[LOAD_PTR_BEFORE_STORE]] [[TMP3]], 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[LOAD_PTR_BEFORE_STORE]] [[TMP3]], 0
+; CHECK-NEXT: ret i32 [[TMP5]]
+;
+ %load = load i32, ptr addrspace(5) %ptr
+ store i32 0, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define i32 @volatile_load_ptr_before_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define i32 @volatile_load_ptr_before_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[VOLATILE_LOAD_PTR_BEFORE_STORE:%.*]] @[[VOLATILE_LOAD_PTR_BEFORE_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[VOLATILE_LOAD_PTR_BEFORE_STORE]] [[TMP3]], 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue [[VOLATILE_LOAD_PTR_BEFORE_STORE]] [[TMP3]], 0
+; CHECK-NEXT: ret i32 [[TMP5]]
+;
+ %load = load volatile i32, ptr addrspace(5) %ptr
+ store i32 0, ptr addrspace(5) %out
+ ret i32 %load
+}
+
+define void @atomic_store_release_ptr_before_store(ptr addrspace(5) %out, ptr addrspace(5) %ptr) {
+; CHECK-LABEL: define void @atomic_store_release_ptr_before_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]], ptr addrspace(5) [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[ATOMIC_STORE_RELEASE_PTR_BEFORE_STORE:%.*]] @[[ATOMIC_STORE_RELEASE_PTR_BEFORE_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[ATOMIC_STORE_RELEASE_PTR_BEFORE_STORE]] [[TMP3]], 0
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store atomic i32 1, ptr addrspace(5) %ptr release, align 4
+ store i32 0, ptr addrspace(5) %out
+ ret void
+}
+
+define void @atomic_store_seq_cst_noalias_ptr_after_store(ptr addrspace(5) noalias %out, ptr addrspace(5) noalias %ptr) {
+; CHECK-LABEL: define void @atomic_store_seq_cst_noalias_ptr_after_store(
+; CHECK-SAME: ptr addrspace(5) noalias [[TMP0:%.*]], ptr addrspace(5) noalias [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP3:%.*]] = call [[ATOMIC_STORE_SEQ_CST_NOALIAS_PTR_AFTER_STORE:%.*]] @[[ATOMIC_STORE_SEQ_CST_NOALIAS_PTR_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison, ptr addrspace(5) [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue [[ATOMIC_STORE_SEQ_CST_NOALIAS_PTR_AFTER_STORE]] [[TMP3]], 0
+; CHECK-NEXT: store i32 [[TMP4]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ store atomic i32 1, ptr addrspace(5) %ptr seq_cst, align 4
+ ret void
+}
+
+define void @fence_release_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @fence_release_after_store(
+; CHECK-SAME: ptr addrspace(5) [[TMP0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP2:%.*]] = call [[FENCE_RELEASE_AFTER_STORE:%.*]] @[[FENCE_RELEASE_AFTER_STORE_BODY:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](ptr addrspace(5) poison)
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue [[FENCE_RELEASE_AFTER_STORE]] [[TMP2]], 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(5) [[TMP0]], align 4
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ fence release
+ ret void
+}
+
+define void @fence_seq_cst_after_store(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @fence_seq_cst_after_store(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: fence seq_cst
+; CHECK-NEXT: ret void
+;
+ store i32 0, ptr addrspace(5) %out
+ fence seq_cst
+ ret void
+}
+
+define void @store_cross_bb(ptr addrspace(5) %out) {
+; CHECK-LABEL: define void @store_cross_bb(
+; CHECK-SAME: ptr addrspace(5) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: store i32 0, ptr addrspace(5) [[OUT]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ store i32 0, ptr addrspace(5) %out
+ br label %exit
+
+exit:
+ ret void
+}
+
+declare void @opaque_read() memory(read)
+declare void @opaque()
+declare void @opaque_rw_argmem(ptr addrspace(5)) memory(argmem: readwrite)
+declare void @llvm.memcpy.p5.p5.i32(ptr addrspace(5), ptr addrspace(5), i32, i1)
+declare void @llvm.memset.p5.i32(ptr addrspace(5), i8, i32, i1)
>From 59295766a14c4324216ad03f6cb88bf866098d52 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me at antoniofrighetto.com>
Date: Tue, 7 Jul 2026 09:28:41 +0200
Subject: [PATCH 2/2] [AMDGPU] Employ MemorySSA when rewriting out arguments
(NFCI)
While transitioning away from MemoryDependenceAnalysis, replace MDA
`getPointerDependencyFrom` query with a reverse traversal of the
memory accesses of the return basic block (MemoryDefs and MemoryUses),
looking for the must-aliasing store to the out argument. The existing
behaviour is meant to be preserved.
---
.../AMDGPU/AMDGPURewriteOutArguments.cpp | 71 ++++++++++++++-----
1 file changed, 55 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index c0802de2763dc..5a3d1e04627dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -44,7 +44,8 @@
#include "AMDGPU.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
@@ -81,7 +82,9 @@ namespace {
class AMDGPURewriteOutArguments : public FunctionPass {
private:
const DataLayout *DL = nullptr;
- MemoryDependenceResults *MDA = nullptr;
+ MemorySSA *MSSA = nullptr;
+ MemorySSAUpdater *MSSAU = nullptr;
+ AAResults *AA = nullptr;
Type *getStoredType(Value &Arg) const;
Type *getOutArgumentType(Argument &Arg) const;
@@ -92,7 +95,8 @@ class AMDGPURewriteOutArguments : public FunctionPass {
AMDGPURewriteOutArguments() : FunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MemoryDependenceWrapperPass>();
+ AU.addRequired<MemorySSAWrapperPass>();
+ AU.addRequired<AAResultsWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
@@ -104,7 +108,7 @@ class AMDGPURewriteOutArguments : public FunctionPass {
INITIALIZE_PASS_BEGIN(AMDGPURewriteOutArguments, DEBUG_TYPE,
"AMDGPU Rewrite Out Arguments", false, false)
-INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
INITIALIZE_PASS_END(AMDGPURewriteOutArguments, DEBUG_TYPE,
"AMDGPU Rewrite Out Arguments", false, false)
@@ -170,6 +174,46 @@ bool AMDGPURewriteOutArguments::doInitialization(Module &M) {
return false;
}
+static StoreInst *findStoreForOutArgument(BasicBlock *BB, Argument *OutArg,
+ MemorySSA &MSSA, AAResults &AA) {
+ MemoryLocation ArgLoc = MemoryLocation::getBeforeOrAfter(OutArg);
+ if (const auto *Accesses = MSSA.getBlockAccesses(BB)) {
+ for (const MemoryAccess &Access : reverse(*Accesses)) {
+ const auto *UseOrDef = dyn_cast<MemoryUseOrDef>(&Access);
+ if (!UseOrDef)
+ continue;
+
+ Instruction *I = UseOrDef->getMemoryInst();
+
+ // Return the must-alias store to the out argument.
+ if (auto *Store = dyn_cast<StoreInst>(I))
+ if (Store->getPointerOperand() == OutArg)
+ return Store;
+
+ if (auto *FI = dyn_cast<FenceInst>(I))
+ if (FI->getOrdering() == AtomicOrdering::Release)
+ continue;
+
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (LI->isAtomic()) {
+ // May-alias reads with monotonic ordering are ignored.
+ if (isStrongerThan(LI->getOrdering(), AtomicOrdering::Monotonic))
+ return nullptr;
+ continue;
+ }
+ }
+
+ // Any other memory access that writes the location prevents the
+ // rewrite.
+ // TODO: Should we handle aliasing reads too?
+ if (isModSet(AA.getModRefInfo(I, ArgLoc)))
+ return nullptr;
+ }
+ }
+
+ return nullptr;
+}
+
bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
@@ -179,7 +223,11 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
AMDGPU::isEntryFunctionCC(F.getCallingConv()))
return false;
- MDA = &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
+ MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
+ AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+
+ MemorySSAUpdater MSSAUpdater(MSSA);
+ MSSAU = &MSSAUpdater;
unsigned ReturnNumRegs = 0;
// Maps an out-argument number to its field index in the return struct.
@@ -254,17 +302,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
for (ReturnInst *RI : Returns) {
BasicBlock *BB = RI->getParent();
- MemDepResult Q = MDA->getPointerDependencyFrom(
- MemoryLocation::getBeforeOrAfter(OutArg), true, BB->end(), BB, RI);
- StoreInst *SI = nullptr;
- if (Q.isDef())
- SI = dyn_cast<StoreInst>(Q.getInst());
-
- // MDA stops at the first may-aliasing store, which need not be to this
- // argument; only fold a store whose pointer is exactly OutArg.
- if (SI && SI->getPointerOperand() != OutArg)
- SI = nullptr;
-
+ StoreInst *SI = findStoreForOutArgument(BB, OutArg, *MSSA, *AA);
if (SI) {
LLVM_DEBUG(dbgs() << "Found out argument store: " << *SI << '\n');
ReplaceableStores.emplace_back(RI, SI);
@@ -291,6 +329,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
}
ValVec.emplace_back(OutArg, ReplVal);
+ MSSAU->removeMemoryAccess(Store.second);
Store.second->eraseFromParent();
}
More information about the llvm-commits
mailing list