[llvm] [AMDGPU] Fix dropped Asyncmarks when new AsyncScore is empty (PR #191894)
Frederick Vu via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 08:59:44 PDT 2026
https://github.com/FrederickVu updated https://github.com/llvm/llvm-project/pull/191894
>From 646ee87213b35dfdd7c0322035e49e5c9083bc88 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Mon, 13 Apr 2026 21:38:41 +0000
Subject: [PATCH 1/2] Merge current AsyncScore with previous score
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 8 ++-
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 51 +++++++++++++++++--
2 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7eb97ca4bf885..0b57f17dfe1a1 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1301,7 +1301,13 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
// limit every time we push a new mark, but that seems like unnecessary work
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
- AsyncMarks.push_back(AsyncScore);
+ CounterValueArray MergedScore = AsyncScore;
+ if (!AsyncMarks.empty()) {
+ const auto &PrevMark = AsyncMarks.back();
+ for (auto T : inst_counter_types(Context->MaxCounter))
+ MergedScore[T] = std::max(AsyncScore[T], PrevMark[T]);
+ }
+ AsyncMarks.push_back(MergedScore);
AsyncScore = {};
LLVM_DEBUG({
dbgs() << "recordAsyncMark:\n" << Inst;
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index b32e883b2b535..24d2344552557 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -486,6 +486,49 @@ epilog:
ret void
}
+define void @double_asyncmark(ptr addrspace(1) %src, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: double_asyncmark:
+; SDAG: ; %bb.0:
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: ds_read_b32 v0, v2
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v[3:4], v0, off
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: double_asyncmark:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: ds_read_b32 v0, v2
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %src, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
; Software pipelined loop with async global-to-LDS and global loads
define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
@@ -509,7 +552,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: v_mov_b32_e32 v13, v8
; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: v_mov_b32_e32 v15, v9
-; SDAG-NEXT: .LBB5_1: ; %loop_body
+; SDAG-NEXT: .LBB6_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s7, v2
; SDAG-NEXT: s_waitcnt vmcnt(1)
@@ -529,7 +572,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(2)
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execnz .LBB5_1
+; SDAG-NEXT: s_cbranch_execnz .LBB6_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: ds_read_b32 v0, v2
@@ -571,7 +614,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: v_mov_b32_e32 v13, v8
; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: v_mov_b32_e32 v15, v9
-; GISEL-NEXT: .LBB5_1: ; %loop_body
+; GISEL-NEXT: .LBB6_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: v_readfirstlane_b32 s6, v2
; GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -591,7 +634,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GISEL-NEXT: s_cbranch_execnz .LBB5_1
+; GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GISEL-NEXT: ds_read_b32 v0, v2
>From b4222a3a9b2b71c23ee0ff6841fb776a0b0a94e1 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Tue, 14 Apr 2026 15:55:53 +0000
Subject: [PATCH 2/2] Replace autos
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 0b57f17dfe1a1..64ff6d2af6b56 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1303,8 +1303,8 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
// loop, which should be sufficient.
CounterValueArray MergedScore = AsyncScore;
if (!AsyncMarks.empty()) {
- const auto &PrevMark = AsyncMarks.back();
- for (auto T : inst_counter_types(Context->MaxCounter))
+ const CounterValueArray &PrevMark = AsyncMarks.back();
+ for (AMDGPU::InstCounterType T : inst_counter_types(Context->MaxCounter))
MergedScore[T] = std::max(AsyncScore[T], PrevMark[T]);
}
AsyncMarks.push_back(MergedScore);
More information about the llvm-commits
mailing list