[llvm] [AMDGPU] BackOffBarrier feature added to gfx1250; Removed incorrect "DS Store drain" check. (PR #179818)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 9 12:07:20 PST 2026
https://github.com/hidekisaito updated https://github.com/llvm/llvm-project/pull/179818
>From e7bda4561864ce59d3c3268b159c4a5658b93572 Mon Sep 17 00:00:00 2001
From: Hideki Saito <hidekido at amd.com>
Date: Wed, 4 Feb 2026 18:39:57 -0500
Subject: [PATCH] [AMDGPU] Removed incorrect "DS Store drain" check.
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 1 +
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 23 +++---
.../AMDGPU/waitcnt-loop-ds-store-barrier.mir | 77 +++++++++++++++++++
3 files changed, 88 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 828f87ccfaf97..1a215f9cfaf1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1935,6 +1935,7 @@ def FeatureISAVersion12 : FeatureSet<
def FeatureISAVersion12_50_Common : FeatureSet<
[FeatureGFX12,
FeatureGFX1250Insts,
+ FeatureBackOffBarrier,
FeatureRequiresAlignedVGPRs,
FeatureCuMode,
Feature1024AddressableVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 9a2a7b4923881..e9b38b3a132d0 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3229,7 +3229,6 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
PreheaderFlushFlags Flags;
bool HasVMemLoad = false;
bool HasVMemStore = false;
- bool SeenDSStoreInLoop = false;
bool UsesVgprLoadedOutsideVMEM = false;
bool UsesVgprLoadedOutsideDS = false;
bool VMemInvalidated = false;
@@ -3240,20 +3239,20 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
DenseSet<MCRegUnit> VgprDefDS;
for (MachineBasicBlock *MBB : ML->blocks()) {
- bool SeenDSStoreInCurrMBB = false;
for (MachineInstr &MI : *MBB) {
if (isVMEMOrFlatVMEM(MI)) {
HasVMemLoad |= MI.mayLoad();
HasVMemStore |= MI.mayStore();
}
- if (mayStoreIncrementingDSCNT(MI))
- SeenDSStoreInCurrMBB = true;
- // Stores postdominated by a barrier will have a wait at the barrier
- // and thus no need to be waited at the loop header. Barrier found
- // later in the same MBB during in-order traversal is used here as a
- // cheaper alternative to postdomination check.
- if (MI.getOpcode() == AMDGPU::S_BARRIER)
- SeenDSStoreInCurrMBB = false;
+ // TODO: Can we relax DSStore check? There may be cases where
+ // these DS stores are drained prior to the end of MBB (or loop).
+ if (mayStoreIncrementingDSCNT(MI)) {
+ // Early exit if both optimizations are invalidated.
+ // Otherwise, set invalid status and continue.
+ if (VMemInvalidated)
+ return Flags;
+ DSInvalidated = true;
+ }
for (const MachineOperand &Op : MI.all_uses()) {
if (Op.isDebug() || !TRI->isVectorRegister(*MRI, Op.getReg()))
continue;
@@ -3324,8 +3323,6 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
}
}
}
- // Accumulate unprotected DS stores from this MBB
- SeenDSStoreInLoop |= SeenDSStoreInCurrMBB;
}
// VMEM flush decision
@@ -3339,7 +3336,7 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
// are not used in the loop.
// DSInvalidated is pre-set to true on non-GFX12+ targets where DS_CNT
// is LGKM_CNT which also tracks FLAT/SMEM.
- if (!DSInvalidated && !SeenDSStoreInLoop && UsesVgprLoadedOutsideDS)
+ if (!DSInvalidated && UsesVgprLoadedOutsideDS)
Flags.FlushDsCnt = true;
return Flags;
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
new file mode 100644
index 0000000000000..5b8b5207b1eee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
@@ -0,0 +1,77 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# Test for the fix that removed the incorrect S_BARRIER check for DS stores.
+# Previously, the code would reset SeenDSStoreInCurrMBB when encountering an
+# S_BARRIER, incorrectly assuming that stores postdominated by a barrier would
+# be waited at the barrier. This was wrong because:
+# 1. S_BARRIER without AutoWaitcntBeforeBarrier does not automatically wait for DS stores to complete
+# 2. S_BARRIER with BackOffBarrier feature does not later flush memory ops by adding ZERO waitcnt
+#
+# This test ensures that when a loop has a DS store followed by S_BARRIER,
+# the preheader flush optimization is NOT applied (no S_WAIT_DSCNT in preheader).
+# The wait should happen inside the loop instead.
+
+---
+# Test: DS store followed by S_BARRIER in loop.
+# DS load in preheader, value used in loop.
+# The preheader should NOT have S_WAIT_DSCNT because SeenDSStoreInLoop = true.
+# Instead, the wait should be inside the loop.
+name: ds_store_barrier_no_preheader_flush
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ ; CHECK-LABEL: name: ds_store_barrier_no_preheader_flush
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; Verify NO S_WAIT_DSCNT in preheader - the wait must be inside the loop
+ ; CHECK-NOT: S_WAIT_DSCNT
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_DSCNT 0
+ ; CHECK-NEXT: $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+ ; With BackOffBarrier, no S_WAIT_DSCNT needed before S_BARRIER
+ ; CHECK-NEXT: S_BARRIER
+ ; CHECK-NEXT: $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1
+ liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+
+ ; Preheader: DS load
+ $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1, %bb.2
+ liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+
+ ; Use DS-loaded value from preheader
+ $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+
+ ; DS store followed by barrier - this should NOT reset SeenDSStoreInCurrMBB
+ DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+ S_BARRIER
+
+ ; Loop control
+ $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list