[llvm] 55e8c5e - [AMDGPU] Mark ASYNCMARK as meta instruction to fix hazard cycle miscounting (#189981)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 10:58:28 PDT 2026
Author: adeshcom14
Date: 2026-04-17T17:58:23Z
New Revision: 55e8c5e0eaea9943c2039a8add9525ed90dad03a
URL: https://github.com/llvm/llvm-project/commit/55e8c5e0eaea9943c2039a8add9525ed90dad03a
DIFF: https://github.com/llvm/llvm-project/commit/55e8c5e0eaea9943c2039a8add9525ed90dad03a.diff
LOG: [AMDGPU] Mark ASYNCMARK as meta instruction to fix hazard cycle miscounting (#189981)
ASYNCMARK emits no hardware code it is used for tracking purpose but was
not marked as meta, causing getNumWaitStates to return 1 and
GCNHazardRecognizer to incorrectly count it as a pipeline cycle.
This patch marks ASYNCMARK as meta-Instruction so it correctly reports 0
wait states.
Fixes: #186878
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
llvm/lib/Target/AMDGPU/SOPInstructions.td
llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 7eb97ca4bf885..f00b99075b022 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -223,6 +223,18 @@ static const unsigned
AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
AMDGPU::S_WAIT_ASYNCCNT};
+// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
+// code but still need to be processed by this pass for async vmcnt tracking.
+static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::ASYNCMARK:
+ case AMDGPU::WAIT_ASYNCMARK:
+ return false;
+ default:
+ return MI.isMetaInstruction();
+ }
+}
+
static bool updateVMCntOnly(const MachineInstr &Inst) {
return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
SIInstrInfo::isFLATGlobal(Inst) || SIInstrInfo::isFLATScratch(Inst);
@@ -1810,7 +1822,7 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
for (auto &II :
make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
- if (II.isMetaInstruction()) {
+ if (isNonWaitcntMetaInst(II)) {
LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
continue;
}
@@ -2059,7 +2071,7 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
for (auto &II :
make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
- if (II.isMetaInstruction()) {
+ if (isNonWaitcntMetaInst(II)) {
LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
continue;
}
@@ -2455,7 +2467,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(
LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
setForceEmitWaitcnt();
- assert(!MI.isMetaInstruction());
+ assert(!isNonWaitcntMetaInst(MI));
AMDGPU::Waitcnt Wait;
const unsigned Opc = MI.getOpcode();
@@ -3308,7 +3320,7 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
E = Block.instr_end();
Iter != E; ++Iter) {
MachineInstr &Inst = *Iter;
- if (Inst.isMetaInstruction())
+ if (isNonWaitcntMetaInst(Inst))
continue;
// Track pre-existing waitcnts that were added in earlier iterations or by
// the memory legalizer.
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index de131b6cdf44d..eb7ad8bcb04ce 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1731,10 +1731,12 @@ let SubtargetPredicate = HasAsyncMark in {
def ASYNCMARK : SPseudoInstSI<(outs), (ins),
[(int_amdgcn_asyncmark)]> {
let maybeAtomic = 0;
+ let isMeta = 1;
}
def WAIT_ASYNCMARK : SOPP_Pseudo <"", (ins s16imm:$simm16), "$simm16",
[(int_amdgcn_wait_asyncmark timm:$simm16)]> {
let maybeAtomic = 0;
+ let isMeta = 1;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
index c6028497c941f..184fd61c355f0 100644
--- a/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/async-buffer-loads.ll
@@ -7,12 +7,13 @@ define float @raw.buffer.load(<4 x i32> inreg %rsrc, ptr addrspace(3) inreg %lds
; CHECK: ; %bb.0: ; %main_body
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 m0, s20
-; CHECK-NEXT: v_mov_b32_e32 v0, s20
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
+; CHECK-NEXT: v_mov_b32_e32 v0, s20
; CHECK-NEXT: ; wait_asyncmark(1)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
@@ -34,12 +35,13 @@ define float @raw.ptr.buffer.load(ptr addrspace(8) inreg %rsrc, ptr addrspace(3)
; CHECK: ; %bb.0: ; %main_body
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 m0, s20
-; CHECK-NEXT: v_mov_b32_e32 v0, s20
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:4 glc lds
; CHECK-NEXT: ; asyncmark
; CHECK-NEXT: buffer_load_dword off, s[16:19], 0 offset:8 slc lds
+; CHECK-NEXT: v_mov_b32_e32 v0, s20
; CHECK-NEXT: ; wait_asyncmark(1)
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_read_b32 v0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 2605b135c4a28..4e5ad79b72468 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -319,9 +319,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
+; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT: s_wait_xcnt 0x0
@@ -338,6 +338,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB2_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
@@ -404,9 +405,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v3, s6
; GISEL-NEXT: s_mov_b64 s[6:7], s[2:3]
+; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; GISEL-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index b32e883b2b535..46af269d0a0a6 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -366,10 +366,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_readfirstlane_b32 s4, v2
; SDAG-NEXT: s_mov_b32 m0, s4
-; SDAG-NEXT: v_mov_b32_e32 v5, 0
+; SDAG-NEXT: s_nop 0
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_dword v[0:1], off lds
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: ; asyncmark
@@ -406,10 +407,11 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_readfirstlane_b32 s4, v2
; GISEL-NEXT: s_mov_b32 m0, s4
-; GISEL-NEXT: s_mov_b32 s6, 0
+; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_load_dword v[0:1], off lds
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_dword v[0:1], off lds
+; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: s_mov_b32 s7, 2
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_mov_b32_e32 v6, s7
More information about the llvm-commits
mailing list