[llvm] [AMDGPU] Use alias analysis for SMEM/VMEM WAR waitcnts (PR #206569)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 12:47:55 PDT 2026
https://github.com/robertvirany updated https://github.com/llvm/llvm-project/pull/206569
>From aeb4cd6534ff88f99506b821b5d498e595d78d41 Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Tue, 7 Jul 2026 15:06:24 -0600
Subject: [PATCH 1/3] [AMDGPU] Precommit SMEM to VMEM WAR waitcnt tests
---
.../CodeGen/AMDGPU/smrd-vmem-war-waitcnt.mir | 32 +++++++++++++++++++
1 file changed, 32 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/smrd-vmem-war-waitcnt.mir
diff --git a/llvm/test/CodeGen/AMDGPU/smrd-vmem-war-waitcnt.mir b/llvm/test/CodeGen/AMDGPU/smrd-vmem-war-waitcnt.mir
new file mode 100644
index 0000000000000..4e690f8277273
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/smrd-vmem-war-waitcnt.mir
@@ -0,0 +1,32 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.00 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+--- |
+ target triple = "amdgcn"
+
+ define amdgpu_kernel void @smrd_vmem_may_alias_war(ptr addrspace(1) %src,
+ ptr addrspace(1) %dst) {
+ ret void
+ }
+...
+---
+name: smrd_vmem_may_alias_war
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5, $vgpr0, $vgpr2, $sgpr6_sgpr7, $sgpr8_sgpr9
+
+ ; CHECK-LABEL: name: smrd_vmem_may_alias_war
+ ; CHECK: liveins: $sgpr4_sgpr5, $vgpr0, $vgpr2, $sgpr6_sgpr7, $sgpr8_sgpr9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ ; CHECK-NEXT: $sgpr4 = S_LOAD_DWORD_IMM renamable $sgpr6_sgpr7, 0, 0 :: (dereferenceable load (s32) from %ir.src, addrspace 1)
+ ; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR renamable $vgpr2, renamable $vgpr2, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s32) into %ir.dst, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr4 = S_LOAD_DWORD_IMM renamable $sgpr6_sgpr7, 0, 0 :: (dereferenceable load (s32) from %ir.src, addrspace 1)
+ GLOBAL_STORE_DWORD_SADDR renamable $vgpr2, renamable $vgpr2, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s32) into %ir.dst, addrspace 1)
+ S_ENDPGM 0
+...
>From 290910751362e8c570f748b745d8a21074d39f06 Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Mon, 29 Jun 2026 15:04:37 -0400
Subject: [PATCH 2/3] [AMDGPU] Use alias analysis for SMEM/VMEM WAR waitcnts
SIInsertWaitcnts tracked prior SMEM loads by their exact IR pointer
Value, so it only inserted a wait before a following VMEM store when the
two memory operands used the same Value.
Track the SMEM load instruction instead and use MachineInstr::mayAlias
when checking following stores. This conservatively handles stores that
may alias the SMEM load even when the memory operands do not share the
same IR Value.
Fixed #203610.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 37 +++++++++++++++------
1 file changed, 26 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 22247238d5542..e4ca5b93b6470 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -337,7 +337,7 @@ struct PreheaderFlushFlags {
};
class SIInsertWaitcnts {
- DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
+ SmallPtrSet<MachineInstr *, 8> PendingSMRDLoads;
DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
MachineLoopInfo &MLI;
MachinePostDominatorTree &PDT;
@@ -2418,18 +2418,33 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
// instruction to guarantee the right WAW order.
// 2) If a destination operand that was used by a recent export/store ins,
// add s_waitcnt on exp_cnt to guarantee the WAR order.
-
for (const MachineMemOperand *Memop : MI.memoperands()) {
const Value *Ptr = Memop->getValue();
- if (Memop->isStore()) {
- if (auto It = SLoadAddresses.find(Ptr); It != SLoadAddresses.end()) {
+ unsigned AS = Memop->getAddrSpace();
+
+ bool IsLDSDMARelevantAS =
+ AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::LOCAL_ADDRESS;
+ // TODO: Rely on SIInstrInfo::areMemAccessesTriviallyDisjoint for this
+ // once it uses MMO address spaces to distinguish lowered memory forms.
+ bool IsSMRDWARRelevantAS = AS == AMDGPUAS::FLAT_ADDRESS ||
+ AMDGPU::isExtendedGlobalAddrSpace(AS);
+
+ if (Memop->isStore() && IsSMRDWARRelevantAS) {
+ SmallVector<MachineInstr *, 4> SMRDLoadsToRemove;
+ for (MachineInstr *SLoad : PendingSMRDLoads) {
+ if (!MI.mayAlias(AA, *SLoad, true))
+ continue;
+
Wait.add(SmemAccessCounter, 0);
- if (PDT.dominates(MI.getParent(), It->second))
- SLoadAddresses.erase(It);
+ if (PDT.dominates(MI.getParent(), SLoad->getParent()))
+ SMRDLoadsToRemove.push_back(SLoad);
}
+
+ for (MachineInstr *SLoad : SMRDLoadsToRemove)
+ PendingSMRDLoads.erase(SLoad);
}
- unsigned AS = Memop->getAddrSpace();
- if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
+
+ if (!IsLDSDMARelevantAS)
continue;
// No need to wait before load from VMEM to LDS.
if (TII.mayWriteLDSThroughDMA(MI))
@@ -3090,13 +3105,13 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
continue;
}
- if (TII.isSMRD(Inst)) {
+ if (TII.isSMRD(Inst) && TII.usesLGKM_CNT(Inst)) {
for (const MachineMemOperand *Memop : Inst.memoperands()) {
// No need to handle invariant loads when avoiding WAR conflicts, as
// there cannot be a vector store to the same memory location.
if (!Memop->isInvariant()) {
- const Value *Ptr = Memop->getValue();
- SLoadAddresses.insert(std::pair(Ptr, Inst.getParent()));
+ PendingSMRDLoads.insert(&Inst);
+ break;
}
}
}
>From 155e7fc360a063d2c1ad11adc5afcd02f99de738 Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Thu, 9 Jul 2026 11:15:45 -0600
Subject: [PATCH 3/3] [AMDGPU] Update SMEM/VMEM WAR waitcnt test checks
---
llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll | 2 +-
llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
index 7b6197e3fcb70..7f5d793d001ac 100644
--- a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
@@ -16,10 +16,10 @@ define amdgpu_kernel void @any_extend_vector_inreg_v16i8_to_v4i32(ptr addrspace(
; GFX6-NEXT: s_load_dword s0, s[8:9], 0x8
; GFX6-NEXT: s_mov_b32 s12, s10
; GFX6-NEXT: s_mov_b32 s13, s11
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_byte v0, off, s[12:15], 0 offset:13
; GFX6-NEXT: s_lshr_b32 s3, s5, 16
; GFX6-NEXT: v_mov_b32_e32 v1, s7
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_lshl_b64 s[4:5], s[0:1], 8
; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 16
; GFX6-NEXT: buffer_store_byte v0, off, s[12:15], 0 offset:15
diff --git a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index 38e30cc85603e..cc60135cf0d22 100644
--- a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -236,6 +236,7 @@ define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(ptr
; CI-NEXT: v_readfirstlane_b32 s3, v1
; CI-NEXT: v_mov_b32_e32 v0, 0x63
; CI-NEXT: s_load_dword s12, s[2:3], 0x1
+; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; CI-NEXT: s_load_dword s2, s[2:3], 0x3
; CI-NEXT: s_mov_b32 s4, s0
@@ -256,6 +257,7 @@ define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(ptr
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x4
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v2, v3, s[2:3]
; GFX9-NEXT: s_load_dword s2, s[4:5], 0xc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
More information about the llvm-commits
mailing list