[llvm] [AMDGPU][SIInsertWaitcnts] Fix underflow in expand-waitcnt-profiling when Outstanding is 0 on Pre-GFX12 (PR #202465)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 06:49:23 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/202465
>From b937c695b6a851216e6ff16f18cd32a345c71e82 Mon Sep 17 00:00:00 2001
From: akadutta <Akash.Dutta at amd.com>
Date: Mon, 8 Jun 2026 18:34:16 -0500
Subject: [PATCH 1/3] Fix AMDGPU waitcnt profiling underflow
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 +-
...expand-waitcnt-profiling-no-outstanding.ll | 60 +++++++++++++++++++
2 files changed, 63 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7241c0db726ce..6d9113b7d3c47 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -2001,9 +2001,9 @@ bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
// The EmitWaitcnt callback emits a single waitcnt.
auto EmitExpandedWaitcnt = [&](unsigned Outstanding, unsigned Target,
auto EmitWaitcnt) {
- do {
- EmitWaitcnt(--Outstanding);
- } while (Outstanding > Target);
+ for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
+ EmitWaitcnt(I);
+ EmitWaitcnt(Target);
Modified = true;
};
diff --git a/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
new file mode 100644
index 0000000000000..35eddb5bff6c7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-waitcnt-forcezero < %s | FileCheck --check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-waitcnt-forcezero < %s | FileCheck --check-prefix=GFX12 %s
+
+; Regression test for an unsigned-underflow infinite loop in the Pre-GFX12
+; waitcnt profiling expansion (WaitcntGeneratorPreGFX12::createNewWaitcnt).
+;
+; When -amdgpu-expand-waitcnt-profiling expands a wait into a descending
+; sequence and the number of outstanding operations for a counter is 0, the
+; old do/while computed "--Outstanding", wrapping 0 to UINT_MAX and looping
+; ~4 billion times. -amdgpu-waitcnt-forcezero deterministically requests an
+; all-zero wait even when nothing is outstanding (e.g. before the first
+; instruction), exercising that edge. The GFX12+ generator already guarded
+; against the wrap; this checks the Pre-GFX12 path terminates and emits a
+; single waitcnt(0) for the zero-outstanding counters.
+
+define amdgpu_kernel void @no_outstanding(ptr addrspace(1) %p) #0 {
+; GFX9-LABEL: no_outstanding:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_waitcnt expcnt(0)
+; GFX9-NEXT: global_store_dword v0, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX12-LABEL: no_outstanding:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-NEXT: s_endpgm
+ store i32 0, ptr addrspace(1) %p
+ ret void
+}
+
+attributes #0 = { "amdgpu-expand-waitcnt-profiling" }
>From 9224fb2ffeada4b07ec4516b6a56be32118809dc Mon Sep 17 00:00:00 2001
From: akadutta <Akash.Dutta at amd.com>
Date: Mon, 8 Jun 2026 18:45:35 -0500
Subject: [PATCH 2/3] remove extra comments
---
.../AMDGPU/expand-waitcnt-profiling-no-outstanding.ll | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
index 35eddb5bff6c7..b7cc19c8927de 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling-no-outstanding.ll
@@ -4,15 +4,6 @@
; Regression test for an unsigned-underflow infinite loop in the Pre-GFX12
; waitcnt profiling expansion (WaitcntGeneratorPreGFX12::createNewWaitcnt).
-;
-; When -amdgpu-expand-waitcnt-profiling expands a wait into a descending
-; sequence and the number of outstanding operations for a counter is 0, the
-; old do/while computed "--Outstanding", wrapping 0 to UINT_MAX and looping
-; ~4 billion times. -amdgpu-waitcnt-forcezero deterministically requests an
-; all-zero wait even when nothing is outstanding (e.g. before the first
-; instruction), exercising that edge. The GFX12+ generator already guarded
-; against the wrap; this checks the Pre-GFX12 path terminates and emits a
-; single waitcnt(0) for the zero-outstanding counters.
define amdgpu_kernel void @no_outstanding(ptr addrspace(1) %p) #0 {
; GFX9-LABEL: no_outstanding:
>From fbeea78532549f1fb842f1c645aac009bd5f93ca Mon Sep 17 00:00:00 2001
From: akadutta <Akash.Dutta at amd.com>
Date: Tue, 9 Jun 2026 11:39:26 -0500
Subject: [PATCH 3/3] update lambda to a helper function
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 32 ++++++++-------------
1 file changed, 12 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 6d9113b7d3c47..3cddd3102d567 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -99,6 +99,15 @@ static unsigned getWaitCountMax(const AMDGPU::HardwareLimits &Limits,
}
}
+template <typename EmitWaitcntFn>
+static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
+ EmitWaitcntFn &&EmitWaitcnt) {
+ // Emit waitcnts from (Outstanding - 1) down to Target.
+ for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
+ EmitWaitcnt(I);
+ EmitWaitcnt(Target);
+}
+
/// Integer IDs used to track vector memory locations we may have to wait on.
/// Encoded as u16 chunks:
///
@@ -1996,17 +2005,6 @@ bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
bool Modified = false;
const DebugLoc &DL = Block.findDebugLoc(It);
- // Helper to emit expanded waitcnt sequence for profiling.
- // Emits waitcnts from (Outstanding-1) down to Target.
- // The EmitWaitcnt callback emits a single waitcnt.
- auto EmitExpandedWaitcnt = [&](unsigned Outstanding, unsigned Target,
- auto EmitWaitcnt) {
- for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
- EmitWaitcnt(I);
- EmitWaitcnt(Target);
- Modified = true;
- };
-
// Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
// single instruction while VScnt has its own instruction.
if (Wait.hasWaitExceptStoreCnt()) {
@@ -2044,6 +2042,7 @@ bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT))
.addImm(AMDGPU::encodeWaitcnt(IV, W));
});
+ Modified = true;
}
}
} else {
@@ -2074,6 +2073,7 @@ bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
.addReg(AMDGPU::SGPR_NULL, RegState::Undef)
.addImm(Count);
});
+ Modified = true;
} else {
[[maybe_unused]] auto SWaitInst =
BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_VSCNT))
@@ -2404,15 +2404,6 @@ bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
bool Modified = false;
const DebugLoc &DL = Block.findDebugLoc(It);
- // Helper to emit expanded waitcnt sequence for profiling.
- auto EmitExpandedWaitcnt = [&](unsigned Outstanding, unsigned Target,
- auto EmitWaitcnt) {
- for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
- EmitWaitcnt(I);
- EmitWaitcnt(Target);
- Modified = true;
- };
-
// For GFX12+, we use separate wait instructions, which makes expansion
// simpler
if (ExpandWaitcntProfiling) {
@@ -2435,6 +2426,7 @@ bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
BuildMI(Block, It, DL, TII.get(instrsForExtendedCounterTypes[CT]))
.addImm(Val);
});
+ Modified = true;
}
return Modified;
}
More information about the llvm-commits
mailing list