[llvm] [AMDGPU] Fix dropped Asyncmarks when new AsyncScore is empty (PR #191894)

[Frederick Vu via llvm-commits]

https://github.com/FrederickVu created https://github.com/llvm/llvm-project/pull/191894

If we have, say, two consecutive async marks with no intermediate async operations, then waiting on the second async mark would produce no s_waitcnts, potentially leaving async operations before the first async mark outstanding. 

>From 646ee87213b35dfdd7c0322035e49e5c9083bc88 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Mon, 13 Apr 2026 21:38:41 +0000
Subject: [PATCH] Merge current AsyncScore with previous score

---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  8 ++-
 .../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 51 +++++++++++++++++--
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7eb97ca4bf885..0b57f17dfe1a1 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1301,7 +1301,13 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
   // limit every time we push a new mark, but that seems like unnecessary work
   // in practical cases. We do separately truncate the array when processing a
   // loop, which should be sufficient.
-  AsyncMarks.push_back(AsyncScore);
+  CounterValueArray MergedScore = AsyncScore;
+  if (!AsyncMarks.empty()) {
+    const auto &PrevMark = AsyncMarks.back();
+    for (auto T : inst_counter_types(Context->MaxCounter))
+      MergedScore[T] = std::max(AsyncScore[T], PrevMark[T]);
+  }
+  AsyncMarks.push_back(MergedScore);
   AsyncScore = {};
   LLVM_DEBUG({
     dbgs() << "recordAsyncMark:\n" << Inst;
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index b32e883b2b535..24d2344552557 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -486,6 +486,49 @@ epilog:
   ret void
 }
 
+define void @double_asyncmark(ptr addrspace(1) %src, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: double_asyncmark:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT:    v_readfirstlane_b32 s4, v2
+; SDAG-NEXT:    s_mov_b32 m0, s4
+; SDAG-NEXT:    s_nop 0
+; SDAG-NEXT:    global_load_dword v[0:1], off lds
+; SDAG-NEXT:    ; asyncmark
+; SDAG-NEXT:    ; asyncmark
+; SDAG-NEXT:    ; wait_asyncmark(0)
+; SDAG-NEXT:    s_waitcnt vmcnt(0)
+; SDAG-NEXT:    ds_read_b32 v0, v2
+; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-NEXT:    global_store_dword v[3:4], v0, off
+; SDAG-NEXT:    s_waitcnt vmcnt(0)
+; SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: double_asyncmark:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    v_readfirstlane_b32 s4, v2
+; GISEL-NEXT:    s_mov_b32 m0, s4
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    global_load_dword v[0:1], off lds
+; GISEL-NEXT:    ; asyncmark
+; GISEL-NEXT:    ; asyncmark
+; GISEL-NEXT:    ; wait_asyncmark(0)
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    ds_read_b32 v0, v2
+; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-NEXT:    global_store_dword v[3:4], v0, off
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %src, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+  call void @llvm.amdgcn.asyncmark()
+  call void @llvm.amdgcn.asyncmark()
+  call void @llvm.amdgcn.wait.asyncmark(i16 0)
+  %val = load i32, ptr addrspace(3) %lds
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
+
 ; Software pipelined loop with async global-to-LDS and global loads
 
 define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
@@ -509,7 +552,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
 ; SDAG-NEXT:    v_mov_b32_e32 v13, v8
 ; SDAG-NEXT:    s_waitcnt vmcnt(1)
 ; SDAG-NEXT:    v_mov_b32_e32 v15, v9
-; SDAG-NEXT:  .LBB5_1: ; %loop_body
+; SDAG-NEXT:  .LBB6_1: ; %loop_body
 ; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SDAG-NEXT:    v_readfirstlane_b32 s7, v2
 ; SDAG-NEXT:    s_waitcnt vmcnt(1)
@@ -529,7 +572,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
 ; SDAG-NEXT:    ; asyncmark
 ; SDAG-NEXT:    ; wait_asyncmark(2)
 ; SDAG-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT:    s_cbranch_execnz .LBB5_1
+; SDAG-NEXT:    s_cbranch_execnz .LBB6_1
 ; SDAG-NEXT:  ; %bb.2: ; %epilog
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:    ds_read_b32 v0, v2
@@ -571,7 +614,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
 ; GISEL-NEXT:    v_mov_b32_e32 v13, v8
 ; GISEL-NEXT:    s_waitcnt vmcnt(1)
 ; GISEL-NEXT:    v_mov_b32_e32 v15, v9
-; GISEL-NEXT:  .LBB5_1: ; %loop_body
+; GISEL-NEXT:  .LBB6_1: ; %loop_body
 ; GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s6, v2
 ; GISEL-NEXT:    s_waitcnt vmcnt(1)
@@ -591,7 +634,7 @@ define void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspac
 ; GISEL-NEXT:    ; asyncmark
 ; GISEL-NEXT:    ; wait_asyncmark(2)
 ; GISEL-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GISEL-NEXT:    s_cbranch_execnz .LBB5_1
+; GISEL-NEXT:    s_cbranch_execnz .LBB6_1
 ; GISEL-NEXT:  ; %bb.2: ; %epilog
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:    ds_read_b32 v0, v2