[llvm] [AMDGPU] Do not reset AsyncScore when recording an async mark (PR #213144)
Jake Daly via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 16:57:20 PDT 2026
https://github.com/jakemdaly updated https://github.com/llvm/llvm-project/pull/213144
>From f92481c181df95c36c021bbbc7ff5f1a5e09db4c Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 14:03:25 -0700
Subject: [PATCH 1/6] [AMDGPU] Add test case.
---
.../CodeGen/AMDGPU/asyncmark-consecutive.mir | 31 +++++++++++++++++++
1 file changed, 31 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
new file mode 100644
index 0000000000000..ec53c009f90a8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+
+---
+name: asyncmark_consecutive
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 8
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2
+
+ ; CHECK-LABEL: name: asyncmark_consecutive
+ ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ ; CHECK-NEXT: $m0 = S_MOV_B32 0
+ ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: ASYNCMARK
+ ; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ ; CHECK-NEXT: S_ENDPGM 0
+ $m0 = S_MOV_B32 0
+ GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
+ ASYNCMARK
+ ASYNCMARK
+ WAIT_ASYNCMARK 0
+ renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
+ S_ENDPGM 0
+...
>From bd6357e49f0a65b0d101f3da29fb587a9ed4b711 Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 14:12:12 -0700
Subject: [PATCH 2/6] [AMDGPU] Do not reset AsyncScore when recording an async
mark
AsyncScore is a snapshot of the counter scores used by async operations, which recordAsyncMark stores into AsyncMarks. Like the other scores tracked by the brackets, these snapshots need to be monotonically increasing: determineAsyncWait indexes into AsyncMarks and uses the selected entry directly to compute the wait, so each mark has to describe the state of every async operation issued before it, not just those issued since the previous mark.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 1 -
llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index b3390a8f5fac0..64188ad64061a 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1094,7 +1094,6 @@ void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
// in practical cases. We do separately truncate the array when processing a
// loop, which should be sufficient.
AsyncMarks.push_back(AsyncScore);
- AsyncScore = {};
LLVM_DEBUG({
dbgs() << "recordAsyncMark:\n" << Inst;
for (const auto &Mark : AsyncMarks) {
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
index ec53c009f90a8..734a1bc4dfed4 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
@@ -19,6 +19,7 @@ body: |
; CHECK-NEXT: ASYNCMARK
; CHECK-NEXT: ASYNCMARK
; CHECK-NEXT: WAIT_ASYNCMARK 0
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_ENDPGM 0
$m0 = S_MOV_B32 0
>From 69631a0cc643a32b3ff443a34c1ad35305833836 Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 15:47:18 -0700
Subject: [PATCH 3/6] Move test to a pre-existing file; add comment.
---
.../CodeGen/AMDGPU/asyncmark-consecutive.mir | 32 -------------
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 46 +++++++++++++++++++
2 files changed, 46 insertions(+), 32 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir b/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
deleted file mode 100644
index 734a1bc4dfed4..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-consecutive.mir
+++ /dev/null
@@ -1,32 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
-
----
-name: asyncmark_consecutive
-tracksRegLiveness: true
-machineFunctionInfo:
- occupancy: 8
-body: |
- bb.0:
- liveins: $vgpr0_vgpr1, $vgpr2
-
- ; CHECK-LABEL: name: asyncmark_consecutive
- ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
- ; CHECK-NEXT: $m0 = S_MOV_B32 0
- ; CHECK-NEXT: GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ; CHECK-NEXT: ASYNCMARK
- ; CHECK-NEXT: ASYNCMARK
- ; CHECK-NEXT: WAIT_ASYNCMARK 0
- ; CHECK-NEXT: S_WAITCNT .Vmcnt_0
- ; CHECK-NEXT: renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
- ; CHECK-NEXT: S_ENDPGM 0
- $m0 = S_MOV_B32 0
- GLOBAL_LOAD_LDS_DWORD $vgpr0_vgpr1, 0, 0, 1, implicit $m0, implicit $exec :: (load (s32), addrspace 1), (store (s32), addrspace 3)
- ASYNCMARK
- ASYNCMARK
- WAIT_ASYNCMARK 0
- renamable $vgpr0 = DS_READ_B32_gfx9 killed $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
- S_ENDPGM 0
-...
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 7c8b2ce209434..973e5626cd08e 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -677,3 +677,49 @@ epilog:
ret void
}
+
+; The second mark records only the async operations issued since the first, so it
+; snapshots an empty score and wait.asyncmark(0) emits no vmcnt wait even though
+; it removes both marks.
+
+define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ds_read_b32 v0, v2
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v[3:4], v0, off
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ds_read_b32 v0, v2
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
>From 3acd456e2652e4fbdf771b81b9725a6c7ffb4208 Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 15:50:19 -0700
Subject: [PATCH 4/6] Show updated test with fix.
---
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 973e5626cd08e..334711bab4805 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -693,6 +693,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -709,6 +710,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
>From 10c55238f8bad5e72753591df060dae8114a827c Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 17:09:08 -0700
Subject: [PATCH 5/6] Add a test with wait.asyncmark(1). Add tests to
gfx12plus.
---
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 93 +++++++++++++++++++
.../test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 60 +++++++++++-
2 files changed, 151 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index f44294014da19..87557860504f5 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -576,3 +576,96 @@ epilog:
ret void
}
+
+; The second mark records no async operation of its own, but wait.asyncmark(0)
+; removes both marks and so must still wait for the transfer tracked by the first.
+
+define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: ds_load_b32 v0, v2
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: ds_load_b32 v0, v2
+; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
+
+; wait.asyncmark(1) removes the first two marks, so it must wait for the transfer
+; tracked by the first while leaving the one after the empty mark in flight.
+
+define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks_wait1:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; SDAG-NEXT: v_add_nc_u32_e32 v3, 4, v2
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ds_load_b32 v0, v2
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks_wait1:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v2
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ds_load_b32 v0, v2
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 334711bab4805..34aeeca20b340 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -693,7 +693,6 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
-; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -710,7 +709,6 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
-; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -725,3 +723,61 @@ entry:
store i32 %val, ptr addrspace(1) %out
ret void
}
+
+; wait.asyncmark(1) removes the first two marks, so it must wait for the DMA
+; tracked by the first while leaving the DMA after the empty mark in flight.
+
+define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
+; SDAG-LABEL: consecutive_asyncmarks_wait1:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_add_u32_e32 v5, 4, v2
+; SDAG-NEXT: v_readfirstlane_b32 s4, v2
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: v_readfirstlane_b32 s4, v5
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: s_mov_b32 m0, s4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: ds_read_b32 v0, v2
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: global_store_dword v[3:4], v0, off
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: consecutive_asyncmarks_wait1:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_add_u32_e32 v5, 4, v2
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: v_readfirstlane_b32 s4, v5
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: s_mov_b32 m0, s4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: ds_read_b32 v0, v2
+; GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-NEXT: global_store_dword v[3:4], v0, off
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.global.load.async.lds(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0, i32 0)
+ call void @llvm.amdgcn.asyncmark()
+ call void @llvm.amdgcn.wait.asyncmark(i16 1)
+ %val = load i32, ptr addrspace(3) %lds
+ store i32 %val, ptr addrspace(1) %out
+ ret void
+}
>From 628037676f3b44ab9356b4f2b1f7689048a3a339 Mon Sep 17 00:00:00 2001
From: Jake Daly <jake.daly at ymail.com>
Date: Thu, 30 Jul 2026 17:09:50 -0700
Subject: [PATCH 6/6] Show changes effect on new tests.
---
llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 4 ++++
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 4 ++++
2 files changed, 8 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 87557860504f5..16af698b46886 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -589,6 +589,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; SDAG-NEXT: s_wait_dscnt 0x0
@@ -603,6 +604,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GISEL-NEXT: s_wait_dscnt 0x0
@@ -635,6 +637,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v0, v2
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: global_store_b32 v[4:5], v0, off
@@ -653,6 +656,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v0, v2
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: global_store_b32 v[6:7], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index 34aeeca20b340..5a86750c453e1 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -693,6 +693,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -709,6 +710,7 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
@@ -743,6 +745,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
; SDAG-NEXT: ds_read_b32 v0, v2
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: global_store_dword v[3:4], v0, off
@@ -764,6 +767,7 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_waitcnt vmcnt(1)
; GISEL-NEXT: ds_read_b32 v0, v2
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: global_store_dword v[3:4], v0, off
More information about the llvm-commits
mailing list