[llvm] [AMDGPU] Mark ASYNCMARK as meta instruction to fix hazard cycle miscounting (PR #189981)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 04:05:15 PDT 2026
https://github.com/adeshcom14 updated https://github.com/llvm/llvm-project/pull/189981
>From a83d23f160fc5ae375ec585beb1e73a8cb367eba Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Wed, 1 Apr 2026 14:34:54 +0000
Subject: [PATCH 1/7] Mark Asyncmark as meta Instr
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 ++++--
llvm/lib/Target/AMDGPU/SOPInstructions.td | 1 +
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7eb97ca4bf885..a896dedcbc0f6 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -2455,7 +2455,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
setForceEmitWaitcnt();
- assert(!MI.isMetaInstruction());
+ assert(!MI.isMetaInstruction() || MI.getOpcode() == AMDGPU::ASYNCMARK);
AMDGPU::Waitcnt Wait;
const unsigned Opc = MI.getOpcode();
@@ -3308,7 +3308,9 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
E = Block.instr_end();
Iter != E; ++Iter) {
MachineInstr &Inst = *Iter;
- if (Inst.isMetaInstruction())
+ // ASYNCMARK is meta instr but needs processing by
+ // generateWaitcntInstBefore and recordAsyncMark for vmcnt tracking.
+ if (Inst.isMetaInstruction() && Inst.getOpcode() != AMDGPU::ASYNCMARK)
continue;
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index de131b6cdf44d..477c0f526236b 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1731,6 +1731,7 @@ let SubtargetPredicate = HasAsyncMark in {
def ASYNCMARK : SPseudoInstSI<(outs), (ins),
[(int_amdgcn_asyncmark)]> {
let maybeAtomic = 0;
+ let isMeta = 1;
}
def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16), "$simm16",
[(int_amdgcn_wait_asyncmark timm:$simm16)]> {
>From 28a19784ab9912478f51b5721158aff0090e7ae0 Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Wed, 1 Apr 2026 14:36:17 +0000
Subject: [PATCH 2/7] Regenerated tests after fix
---
llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
index c6028497c941f..184fd61c355f0 100644
--- a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
@@ -7,12 +7,13 @@ define float @raw.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %lds
; CHECK: ; %bb.0: ; %main_body
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 m0, s20
-; CHECK-NEXT: v_mov_b32_e32 v0, s20
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
+; CHECK-NEXT: v_mov_b32_e32 v0, s20
; CHECK-NEXT: ; wait_asyncmark(1)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
@@ -34,12 +35,13 @@ define float @raw.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace(3)
; CHECK: ; %bb.0: ; %main_body
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 m0, s20
-; CHECK-NEXT: v_mov_b32_e32 v0, s20
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
+; CHECK-NEXT: v_mov_b32_e32 v0, s20
; CHECK-NEXT: ; wait_asyncmark(1)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
>From ff0097bcea934075cdab0cf8a353a5d8d8c148ea Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Tue, 7 Apr 2026 04:30:51 +0000
Subject: [PATCH 3/7] Extract meta instruction skip logic into static helper
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index a896dedcbc0f6..7d65df92452f8 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -223,6 +223,12 @@ static const unsigned
AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
AMDGPU::S_WAIT_ASYNCCNT};
+// ASYNCMARK is a meta instruction that emits no hardware code but still
+// needs to be processed by this pass for async vmcnt tracking.
+static bool shouldSkipForWaitcnt(const MachineInstr &MI) {
+ return MI.isMetaInstruction() && MI.getOpcode() != AMDGPU::ASYNCMARK;
+}
+
static bool updateVMCntOnly(const MachineInstr &Inst) {
return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
SIInstrInfo::isFLATGlobal(Inst) || SIInstrInfo::isFLATScratch(Inst);
@@ -2455,7 +2461,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
setForceEmitWaitcnt();
- assert(!MI.isMetaInstruction() || MI.getOpcode() == AMDGPU::ASYNCMARK);
+ assert(!shouldSkipForWaitcnt(MI));
AMDGPU::Waitcnt Wait;
const unsigned Opc = MI.getOpcode();
@@ -3308,9 +3314,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
E = Block.instr_end();
Iter != E; ++Iter) {
MachineInstr &Inst = *Iter;
- // ASYNCMARK is meta instr but needs processing by
- // generateWaitcntInstBefore and recordAsyncMark for vmcnt tracking.
- if (Inst.isMetaInstruction() && Inst.getOpcode() != AMDGPU::ASYNCMARK)
+ if (shouldSkipForWaitcnt(Inst))
continue;
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
>From b91063860c02437f3228ccbfab3b4cefeb1a0005 Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Fri, 10 Apr 2026 07:08:40 +0000
Subject: [PATCH 4/7] Rename shouldSkipForWaitcnt to
shouldSkipWaitcntInsertionBefore
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7d65df92452f8..b867426aebdfa 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -225,7 +225,7 @@ static const unsigned
// ASYNCMARK is a meta instruction that emits no hardware code but still
// needs to be processed by this pass for async vmcnt tracking.
-static bool shouldSkipForWaitcnt(const MachineInstr &MI) {
+static bool shouldSkipWaitcntInsertionBefore(const MachineInstr &MI) {
return MI.isMetaInstruction() && MI.getOpcode() != AMDGPU::ASYNCMARK;
}
@@ -2461,7 +2461,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
setForceEmitWaitcnt();
- assert(!shouldSkipForWaitcnt(MI));
+ assert(!shouldSkipWaitcntInsertionBefore(MI));
AMDGPU::Waitcnt Wait;
const unsigned Opc = MI.getOpcode();
@@ -3314,7 +3314,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
E = Block.instr_end();
Iter != E; ++Iter) {
MachineInstr &Inst = *Iter;
- if (shouldSkipForWaitcnt(Inst))
+ if (shouldSkipWaitcntInsertionBefore(Inst))
continue;
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
>From 6189965edfe0f805cea5eba1e798518547b25d87 Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Wed, 15 Apr 2026 06:48:17 +0000
Subject: [PATCH 5/7] Mark wait_asyncmark as meta Instr
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 17 +++++++++--------
llvm/lib/Target/AMDGPU/SOPInstructions.td | 1 +
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index b867426aebdfa..9e69c478d3d34 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -223,10 +223,11 @@ static const unsigned
AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
AMDGPU::S_WAIT_ASYNCCNT};
-// ASYNCMARK is a meta instruction that emits no hardware code but still
-// needs to be processed by this pass for async vmcnt tracking.
-static bool shouldSkipWaitcntInsertionBefore(const MachineInstr &MI) {
- return MI.isMetaInstruction() && MI.getOpcode() != AMDGPU::ASYNCMARK;
+// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
+// code but still need to be processed by this pass for async vmcnt tracking.
+static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
+ return MI.isMetaInstruction() && MI.getOpcode() != AMDGPU::ASYNCMARK &&
+ MI.getOpcode() != AMDGPU::WAIT_ASYNCMARK;
}
static bool updateVMCntOnly(const MachineInstr &Inst) {
@@ -1816,7 +1817,7 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
for (auto &II :
make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
- if (II.isMetaInstruction()) {
+ if (isNonWaitcntMetaInst(II)) {
LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
continue;
}
@@ -2065,7 +2066,7 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
for (auto &II :
make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
- if (II.isMetaInstruction()) {
+ if (isNonWaitcntMetaInst(II)) {
LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
continue;
}
@@ -2461,7 +2462,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
setForceEmitWaitcnt();
- assert(!shouldSkipWaitcntInsertionBefore(MI));
+ assert(!isNonWaitcntMetaInst(MI));
AMDGPU::Waitcnt Wait;
const unsigned Opc = MI.getOpcode();
@@ -3314,7 +3315,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
E = Block.instr_end();
Iter != E; ++Iter) {
MachineInstr &Inst = *Iter;
- if (shouldSkipWaitcntInsertionBefore(Inst))
+ if (isNonWaitcntMetaInst(Inst))
continue;
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 477c0f526236b..eb7ad8bcb04ce 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1736,6 +1736,7 @@ def ASYNCMARK : SPseudoInstSI<(outs), (ins),
def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16), "$simm16",
[(int_amdgcn_wait_asyncmark timm:$simm16)]> {
let maybeAtomic = 0;
+ let isMeta = 1;
}
}
>From c43ef06dc4c4177759ed57dff6e1510577e3eba2 Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Wed, 15 Apr 2026 06:48:57 +0000
Subject: [PATCH 6/7] Regenerated test after fix
---
llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 5 +++--
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll | 6 ++++--
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 2605b135c4a28..4e5ad79b72468 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -319,9 +319,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
+; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT: s_wait_xcnt 0x0
@@ -338,6 +338,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB2_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
@@ -404,9 +405,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v3, s6
; GISEL-NEXT: s_mov_b64 s[6:7], s[2:3]
+; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; GISEL-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index b32e883b2b535..46af269d0a0a6 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -366,10 +366,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_readfirstlane_b32 s4, v2
; SDAG-NEXT: s_mov_b32 m0, s4
-; SDAG-NEXT: v_mov_b32_e32 v5, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: ; asyncmark
@@ -406,10 +407,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_readfirstlane_b32 s4, v2
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: s_mov_b32 s6, 0
+; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: s_mov_b32 s7, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v6, s7
>From 3f8b4b7d9841e0a86161d6738311b3b4091daf9c Mon Sep 17 00:00:00 2001
From: Adesh Adikane <aadikane at amd.com>
Date: Fri, 17 Apr 2026 11:02:26 +0000
Subject: [PATCH 7/7] Refactor isNonWaitcntMetaInst to switch-case
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 9e69c478d3d34..f00b99075b022 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -226,8 +226,13 @@ static const unsigned
// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
// code but still need to be processed by this pass for async vmcnt tracking.
static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
- return MI.isMetaInstruction() && MI.getOpcode() != AMDGPU::ASYNCMARK &&
- MI.getOpcode() != AMDGPU::WAIT_ASYNCMARK;
+ switch (MI.getOpcode()) {
+ case AMDGPU::ASYNCMARK:
+ case AMDGPU::WAIT_ASYNCMARK:
+ return false;
+ default:
+ return MI.isMetaInstruction();
+ }
}
static bool updateVMCntOnly(const MachineInstr &Inst) {
More information about the llvm-commits
mailing list