[llvm] [AMDGPU] Do not reset AsyncScore when recording an async mark (PR #213144)
Jake Daly via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 10:14:25 PDT 2026
https://github.com/jakemdaly updated https://github.com/llvm/llvm-project/pull/213144
>From 80cbb81f634c351c272236444cfefb998680d1e1 Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Fri, 31 Jul 2026 10:24:20 -0700
Subject: [PATCH 1/2] [AMDGPU] Add test cases with buggy behavior.
---
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 94 ++++++++++++++++
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 104 ++++++++++++++++++
2 files changed, 198 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index f44294014da19..25d257dec2056 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -576,3 +576,97 @@ epilog:
ret void
}
+
+; The second asyncmark's snapshot of the counters should be the same as the first;
+; when wait.asyncmark(0) is called it should insert a asynccnt before the ds_load.
+
+define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ds_load_b32 v0, v2
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ds_load_b32 v0, v2
+; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Similar to the previous test, except wait.asyncmark(1) should remove the
+; the first two marks, allowing the second DMA to remain in flight (ie.
+; asynccnt 0x1) during the ds_load.
+
+define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks_wait1:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; SDAG-NEXT: v_add_nc_u32_e32 v3, 4, v2
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ds_load_b32 v0, v2
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks_wait1:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v2
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ds_load_b32 v0, v2
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 7c8b2ce209434..987dd167d6f53 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -677,3 +677,107 @@ epilog:
ret void
}
+
+; The second asyncmark's snapshot of the counters should be the same as the first;
+; when wait.asyncmark(0) is called it should insert a vmcnt before the ds_read.
+
+define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ds_read_b32 v0, v2
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v[3:4], v0, off
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ds_read_b32 v0, v2
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; Similar to the previous test, except wait.asyncmark(1) should remove the
+; the first two marks, allowing the second DMA to remain in flight (ie.
+; vmcnt 0x1) during the ds_read.
+
+define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks_wait1:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v5, 4, v2
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: v_readfirstlane_b32 s4, v5
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ds_read_b32 v0, v2
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v[3:4], v0, off
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks_wait1:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v5, 4, v2
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: v_readfirstlane_b32 s4, v5
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ds_read_b32 v0, v2
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
>From 3a7537e1cb374a1c2e35cb8cbd554d39d31ee78b Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Fri, 31 Jul 2026 10:27:09 -0700
Subject: [PATCH 2/2] [AMDGPU] Don't reset async score.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 1 -
llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 4 ++++
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 6 +++++-
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index b3390a8f5fac0..64188ad64061a 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1094,7 +1094,6 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
AsyncMarks.push_back(AsyncScore);
- AsyncScore = {};
LLVM_DEBUG({
dbgs() << "recordAsyncMark:\n" << Inst;
for (const auto &Mark : AsyncMarks) {
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 25d257dec2056..8dba79f10eb41 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -589,6 +589,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -603,6 +604,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -636,6 +638,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: global_store_b32 v[4:5], v0, off
@@ -654,6 +657,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: global_store_b32 v[6:7], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 987dd167d6f53..3d8f0b94c24a7 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -692,6 +692,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -708,6 +709,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -725,7 +727,7 @@ entry:
; Similar to the previous test, except wait.asyncmark(1) should remove the
; the first two marks, allowing the second DMA to remain in flight (ie.
-; vmcnt 0x1) during the ds_read.
+; vmcnt(1)) during the ds_read.
define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
; SDAG-LABEL: consecutive_asyncmarks_wait1:
@@ -743,6 +745,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -764,6 +767,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
More information about the llvm-commits
mailing list