[llvm] [AMDGPU] BackOffBarrier feature added to gfx1250; Removed incorrect "DS Store drain" check. (PR #179818)

via llvm-commits llvm-commits at lists.llvm.org
Mon Feb 9 12:07:20 PST 2026


https://github.com/hidekisaito updated https://github.com/llvm/llvm-project/pull/179818

>From e7bda4561864ce59d3c3268b159c4a5658b93572 Mon Sep 17 00:00:00 2001
From: Hideki Saito <hidekido at amd.com>
Date: Wed, 4 Feb 2026 18:39:57 -0500
Subject: [PATCH] [AMDGPU] Removed incorrect "DS Store drain" check.

---
 llvm/lib/Target/AMDGPU/AMDGPU.td              |  1 +
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   | 23 +++---
 .../AMDGPU/waitcnt-loop-ds-store-barrier.mir  | 77 +++++++++++++++++++
 3 files changed, 88 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 828f87ccfaf97..1a215f9cfaf1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1935,6 +1935,7 @@ def FeatureISAVersion12 : FeatureSet<
 def FeatureISAVersion12_50_Common : FeatureSet<
   [FeatureGFX12,
    FeatureGFX1250Insts,
+   FeatureBackOffBarrier,
    FeatureRequiresAlignedVGPRs,
    FeatureCuMode,
    Feature1024AddressableVGPRs,
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 9a2a7b4923881..e9b38b3a132d0 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3229,7 +3229,6 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
   PreheaderFlushFlags Flags;
   bool HasVMemLoad = false;
   bool HasVMemStore = false;
-  bool SeenDSStoreInLoop = false;
   bool UsesVgprLoadedOutsideVMEM = false;
   bool UsesVgprLoadedOutsideDS = false;
   bool VMemInvalidated = false;
@@ -3240,20 +3239,20 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
   DenseSet<MCRegUnit> VgprDefDS;
 
   for (MachineBasicBlock *MBB : ML->blocks()) {
-    bool SeenDSStoreInCurrMBB = false;
     for (MachineInstr &MI : *MBB) {
       if (isVMEMOrFlatVMEM(MI)) {
         HasVMemLoad |= MI.mayLoad();
         HasVMemStore |= MI.mayStore();
       }
-      if (mayStoreIncrementingDSCNT(MI))
-        SeenDSStoreInCurrMBB = true;
-      // Stores postdominated by a barrier will have a wait at the barrier
-      // and thus no need to be waited at the loop header. Barrier found
-      // later in the same MBB during in-order traversal is used here as a
-      // cheaper alternative to postdomination check.
-      if (MI.getOpcode() == AMDGPU::S_BARRIER)
-        SeenDSStoreInCurrMBB = false;
+      // TODO: Can we relax DSStore check? There may be cases where
+      // these DS stores are drained prior to the end of MBB (or loop).
+      if (mayStoreIncrementingDSCNT(MI)) {
+        // Early exit if both optimizations are invalidated.
+        // Otherwise, set invalid status and continue.
+        if (VMemInvalidated)
+          return Flags;
+        DSInvalidated = true;
+      }
       for (const MachineOperand &Op : MI.all_uses()) {
         if (Op.isDebug() || !TRI->isVectorRegister(*MRI, Op.getReg()))
           continue;
@@ -3324,8 +3323,6 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
         }
       }
     }
-    // Accumulate unprotected DS stores from this MBB
-    SeenDSStoreInLoop |= SeenDSStoreInCurrMBB;
   }
 
   // VMEM flush decision
@@ -3339,7 +3336,7 @@ SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
   // are not used in the loop.
   // DSInvalidated is pre-set to true on non-GFX12+ targets where DS_CNT
   // is LGKM_CNT which also tracks FLAT/SMEM.
-  if (!DSInvalidated && !SeenDSStoreInLoop && UsesVgprLoadedOutsideDS)
+  if (!DSInvalidated && UsesVgprLoadedOutsideDS)
     Flags.FlushDsCnt = true;
 
   return Flags;
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
new file mode 100644
index 0000000000000..5b8b5207b1eee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
@@ -0,0 +1,77 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# Test for the fix that removed the incorrect S_BARRIER check for DS stores.
+# Previously, the code would reset SeenDSStoreInCurrMBB when encountering an
+# S_BARRIER, incorrectly assuming that stores postdominated by a barrier would
+# be waited at the barrier. This was wrong because:
+# 1. S_BARRIER without AutoWaitcntBeforeBarrier does not automatically wait for DS stores to complete
+# 2. S_BARRIER with BackOffBarrier feature does not later flush memory ops by adding ZERO waitcnt
+#
+# This test ensures that when a loop has a DS store followed by S_BARRIER,
+# the preheader flush optimization is NOT applied (no S_WAIT_DSCNT in preheader).
+# The wait should happen inside the loop instead.
+
+---
+# Test: DS store followed by S_BARRIER in loop.
+# DS load in preheader, value used in loop.
+# The preheader should NOT have S_WAIT_DSCNT because SeenDSStoreInLoop = true.
+# Instead, the wait should be inside the loop.
+name: ds_store_barrier_no_preheader_flush
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  ; CHECK-LABEL: name: ds_store_barrier_no_preheader_flush
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+  ; Verify NO S_WAIT_DSCNT in preheader - the wait must be inside the loop
+  ; CHECK-NOT:    S_WAIT_DSCNT
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_WAIT_DSCNT 0
+  ; CHECK-NEXT:   $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+  ; With BackOffBarrier, no S_WAIT_DSCNT needed before S_BARRIER
+  ; CHECK-NEXT:   S_BARRIER
+  ; CHECK-NEXT:   $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit $scc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+
+    ; Preheader: DS load
+    $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.1, %bb.2
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+
+    ; Use DS-loaded value from preheader
+    $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+
+    ; DS store followed by barrier - this should NOT reset SeenDSStoreInCurrMBB
+    DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+    S_BARRIER
+
+    ; Loop control
+    $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+    S_CBRANCH_SCC1 %bb.1, implicit $scc
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM 0
+...



More information about the llvm-commits mailing list