[llvm] [AMDGPU] Skip s_delay_alu for gfx1250 WMMA C-reuse chains (PR #214101)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 16:56:57 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Petar Jovanovic (petar-jovanovic)
<details>
<summary>Changes</summary>
Consecutive gfx1250 WMMA/SWMMAC ops accumulating into the same matrix C register reuse the accumulator in place, so the tied srcC read is omitted and no delay is needed. AMDGPUInsertDelayAlu did not model this and emitted an s_delay_alu that stalls the reuse chain.
Detect a C-reuse edge (tied srcC overlaps the previous XDL WMMA's dest, with no intervening instruction) and skip the delay for that operand. The change is state-neutral and safe, since GFX12+ interlocks the matrix C hazard in hardware.
---
Full diff: https://github.com/llvm/llvm-project/pull/214101.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp (+23)
- (modified) llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir (+44)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 21de56463eae2..cc3eb2af86f36 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -360,6 +360,10 @@ class AMDGPUInsertDelayAlu {
// FIXME: 0 is a valid register unit.
MCRegUnit LastSGPRFromVALU = static_cast<MCRegUnit>(0);
+
+ // Destination of the preceding XDL WMMA, for GFX1250 C-reuse detection.
+ Register PrevWMMAVDst;
+
// Iterate over the contents of bundles, but don't emit any instructions
// inside a bundle.
for (auto &MI : MBB.instrs()) {
@@ -390,6 +394,10 @@ class AMDGPUInsertDelayAlu {
State = DelayState();
} else if (Type != OTHER) {
DelayInfo Delay;
+ // GFX1250 C-reuse: back-to-back WMMAs into the same C register forward
+ // the accumulator in place, so the tied srcC read has no dependency.
+ bool IsWMMACReuse = ST->hasGFX1250Insts() && PrevWMMAVDst.isValid() &&
+ SII->isXDLWMMA(MI);
// TODO: Scan implicit uses too?
for (const auto &Op : MI.explicit_uses()) {
if (Op.isReg()) {
@@ -398,6 +406,10 @@ class AMDGPUInsertDelayAlu {
// ignore this operand.
if (MI.getOpcode() == AMDGPU::V_WRITELANE_B32 && Op.isTied())
continue;
+ // Skip the tied srcC of a GFX1250 C-reuse edge.
+ if (IsWMMACReuse && Op.isTied() &&
+ TRI->regsOverlap(Op.getReg(), PrevWMMAVDst))
+ continue;
for (MCRegUnit Unit : TRI->regunits(Op.getReg())) {
auto It = State.find(Unit);
if (It != State.end()) {
@@ -444,6 +456,17 @@ class AMDGPUInsertDelayAlu {
// twice?
State.advance(Type, Cycles);
+ // Track the preceding XDL WMMA's dst for C-reuse; reset on anything else.
+ if (ST->hasGFX1250Insts()) {
+ if (SII->isXDLWMMA(MI)) {
+ const MachineOperand *VDst =
+ SII->getNamedOperand(MI, AMDGPU::OpName::vdst);
+ PrevWMMAVDst = VDst ? VDst->getReg() : Register();
+ } else {
+ PrevWMMAVDst = Register();
+ }
+ }
+
LLVM_DEBUG(dbgs() << " State after " << MI; State.dump(TRI););
}
diff --git a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir
index bafa3c4f76cae..b4bb8dfd33524 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir
@@ -65,3 +65,47 @@ body: |
$vgpr12 = V_EXP_F32_e32 $vgpr12, implicit $exec, implicit $mode
$vgpr13 = V_ADD_U32_e32 $vgpr13, $vgpr8, implicit $exec
...
+
+name: wmma_xdl_c_reuse_no_delay
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}wmma_xdl_c_reuse_no_delay:
+ ; CHECK: %bb.0:
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[0:7], v[0:7], v[8:15]
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[16:23], v[16:23], v[8:15]
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+...
+
+name: wmma_xdl_c_reuse_chain_no_delay
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}wmma_xdl_c_reuse_chain_no_delay:
+ ; CHECK: %bb.0:
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[0:7], v[0:7], v[8:15]
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[16:23], v[16:23], v[8:15]
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[24:31], v[24:31], v[8:15]
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+...
+
+name: wmma_xdl_no_c_reuse_when_interrupted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}wmma_xdl_no_c_reuse_when_interrupted:
+ ; CHECK: %bb.0:
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[0:7], v[0:7], v[8:15]
+ ; CHECK-NEXT: v_add_nc_u32_e32 v16, v16, v17
+ ; CHECK-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+ ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[24:31], v[24:31], v[8:15]
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16, $vgpr17
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr16 = V_ADD_U32_e32 $vgpr16, $vgpr17, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/214101
More information about the llvm-commits
mailing list