[llvm] [AMDGPU] SIInsertWaitcnts: rebase async marks into the merged frame at CFG joins (PR #211688)
Yoonseo Choi via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 20:29:26 PDT 2026
https://github.com/yoonseoch updated https://github.com/llvm/llvm-project/pull/211688
>From 85f2de488b5693e63693126369b2223ad0351f3f Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Thu, 23 Jul 2026 21:55:14 +0000
Subject: [PATCH 1/2] [AMDGPU] SIInsertWaitcnts: rebase async marks into the
merged frame at CFG joins
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 37 ++--
.../asyncmark-merge-rebase-pregfx12.mir | 186 ++++++++++++++++++
2 files changed, 208 insertions(+), 15 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 22247238d5542..3402b9ba9f5bc 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -2796,21 +2796,28 @@ bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
});
// Merge element-wise using the existing mergeScore function and the
- // appropriate MergeInfo for each counter type. Iterate only while we have
- // elements in both vectors.
- unsigned OtherSize = OtherMarks.size();
- unsigned OurSize = AsyncMarks.size();
- unsigned MergeCount = std::min(OtherSize, OurSize);
- // OtherMarks is empty -> OtherSize == 0 -> MergeCount == 0.
- // Our existing marks are the conservative result; return early to avoid
- // passing MergeCount == 0 to seq_inclusive which asserts Begin <= End.
- if (MergeCount == 0)
- return StrictDom;
- for (auto Idx : seq_inclusive<unsigned>(1, MergeCount)) {
- for (auto T : inst_counter_types(Context->MaxCounter)) {
- StrictDom |= mergeScore(MergeInfos[T], AsyncMarks[OurSize - Idx][T],
- OtherMarks[OtherSize - Idx][T]);
- }
+ // appropriate MergeInfo for each counter type. Rebase and merge EVERY
+ // surviving mark into the new (widened) frame, aligned by recency from the
+ // back. A mark with no counterpart on the other side merges against the
+ // zero/identity mark: mergeScore still applies MyShift, so the mark is
+ // re-expressed in the new frame instead of being left with a stale
+ // (pre-merge) score.
+ const unsigned OtherSize = OtherMarks.size();
+ const unsigned OurSize = AsyncMarks.size();
+ // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
+ // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
+ // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
+ // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
+ // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
+ assert(OurSize >= 1 &&
+ "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
+
+ for (auto Idx : seq_inclusive<unsigned>(1, OurSize)) {
+ const CounterValueArray &OtherMark =
+ Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
+ for (auto T : inst_counter_types(Context->MaxCounter))
+ StrictDom |=
+ mergeScore(MergeInfos[T], AsyncMarks[OurSize - Idx][T], OtherMark[T]);
}
LLVM_DEBUG({
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
new file mode 100644
index 0000000000000..32c4b7c90ca26
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
@@ -0,0 +1,186 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+# Regression test for WaitcntBrackets::mergeAsyncMarks rebasing surviving async
+# marks into the merged (widened) counter frame at a CFG join.
+#
+# The join (bb.3) has two predecessors:
+# - an async-mark predecessor: one async LDS DMA + ASYNCMARK, so one async mark
+# recorded at LOAD_CNT score 1 (small frame, UB = 1).
+# - a load predecessor: several async LDS DMAs with no ASYNCMARK. These stay
+# outstanding (they write LDS, no VGPR result) and extend the frame to UB = 4,
+# but contribute no async marks.
+#
+# si-insert-waitcnts visits the branch target (bb.2) before the fall-through
+# (bb.1), so whichever of those carries the async mark seeds bb.3's incoming
+# state and the other side is merged in with an empty AsyncMarks list.
+#
+# async_mark_pred_merged_first (bb.2 -> bb.1): the async-mark predecessor seeds the incoming
+# state, then the load predecessor (empty OtherMarks) widens LOAD_CNT to 4. The
+# surviving mark must be shifted from score 1 to score 4. Without the fix
+# mergeAsyncMarks returned early and left the stale score 1, so WAIT_ASYNCMARK 0
+# lowered to s_waitcnt vmcnt(3) (4 - 1) and failed to drain the async LDS DMA.
+# With the fix the mark is rebased and it lowers to s_waitcnt vmcnt(0).
+#
+# load_pred_merged_first (bb.1 -> bb.2): mirror ordering (load predecessor seeds the incoming
+# state). Here the async mark arrives via the non-empty OtherMarks path, which
+# was always rebased, so this case is correct with or without the fix and acts
+# as a control.
+
+---
+name: async_mark_pred_merged_first
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: async_mark_pred_merged_first
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0
+ ; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; fall-through: load predecessor, merged second with empty OtherMarks
+ bb.1:
+ successors: %bb.3
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ $m0 = S_MOV_B32 0
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ S_BRANCH %bb.3
+
+ ; branch target: async-mark predecessor, merged first (seeds incoming state)
+ bb.2:
+ successors: %bb.3
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ $m0 = S_MOV_B32 0
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK
+ S_BRANCH %bb.3
+
+ bb.3:
+ liveins: $vgpr2
+
+ WAIT_ASYNCMARK 0
+ renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ S_ENDPGM 0
+...
+
+---
+name: load_pred_merged_first
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: load_pred_merged_first
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ ; CHECK-NEXT: S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0
+ ; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $sgpr0, $sgpr1, $vgpr0_vgpr1, $vgpr2
+
+ S_CMP_LG_U32 $sgpr0, $sgpr1, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.2, implicit killed $scc
+
+ ; fall-through: async-mark predecessor, merged second (non-empty OtherMarks)
+ bb.1:
+ successors: %bb.3
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ $m0 = S_MOV_B32 0
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK
+ S_BRANCH %bb.3
+
+ ; branch target: load predecessor, merged first (seeds incoming state)
+ bb.2:
+ successors: %bb.3
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ $m0 = S_MOV_B32 0
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ S_BRANCH %bb.3
+
+ bb.3:
+ liveins: $vgpr2
+
+ WAIT_ASYNCMARK 0
+ renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ S_ENDPGM 0
+...
>From be0a0a4a0c82ac24b7c34b7e4f0dd80bd3ab42f5 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Thu, 23 Jul 2026 22:35:02 +0000
Subject: [PATCH 2/2] Update comments
---
.../AMDGPU/asyncmark-merge-rebase-pregfx12.mir | 18 +++++++-----------
1 file changed, 7 insertions(+), 11 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
index 32c4b7c90ca26..37f91ede1d76f 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-merge-rebase-pregfx12.mir
@@ -15,17 +15,13 @@
# (bb.1), so whichever of those carries the async mark seeds bb.3's incoming
# state and the other side is merged in with an empty AsyncMarks list.
#
-# async_mark_pred_merged_first (bb.2 -> bb.1): the async-mark predecessor seeds the incoming
-# state, then the load predecessor (empty OtherMarks) widens LOAD_CNT to 4. The
-# surviving mark must be shifted from score 1 to score 4. Without the fix
-# mergeAsyncMarks returned early and left the stale score 1, so WAIT_ASYNCMARK 0
-# lowered to s_waitcnt vmcnt(3) (4 - 1) and failed to drain the async LDS DMA.
-# With the fix the mark is rebased and it lowers to s_waitcnt vmcnt(0).
-#
-# load_pred_merged_first (bb.1 -> bb.2): mirror ordering (load predecessor seeds the incoming
-# state). Here the async mark arrives via the non-empty OtherMarks path, which
-# was always rebased, so this case is correct with or without the fix and acts
-# as a control.
+# bb.2 seeds the incoming state of bb.3. Next, bb.1 is visisted and merges itself
+# as other to the incoming state. UB of LOAD_CNT is updated to 4 = max(1,4).
+# If mergeScore is not called because bb.1 has no async marks,
+# bb.2's Score 1 (ASYNCMAR) is not rebased to new UB 4. The vmcnt value finally
+# becomes 0 (4 - 1). If score 1 is rebased to 4 (1 + (4 - 1)), vmcnt
+# correctly becomes 0.
+# Previously, mergeScore was not called when other block had no asyncmark.
---
name: async_mark_pred_merged_first
More information about the llvm-commits
mailing list