[llvm] [AMDGPU] Increment VA_VDST twice for each VOP3PX2 instruction (PR #196353)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 02:24:32 PDT 2026
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/196353
>From 19aaac2f2f1b4c80dcb05d49f4847353128d9753 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Thu, 7 May 2026 12:51:09 +0100
Subject: [PATCH 1/3] Precommit tests
---
.../AMDGPU/expert_scheduling_gfx1250.mir | 29 +++++++++++++++++++
1 file changed, 29 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
new file mode 100644
index 0000000000000..5647fef7d4302
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
@@ -0,0 +1,29 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs -amdgpu-expert-scheduling-mode -run-pass si-insert-waitcnts %s -o - -verify-machineinstrs | FileCheck %s
+
+---
+name: wmma_scale
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: wmma_scale
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT 0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16, $vgpr16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_2
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr8_vgpr9, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAIT_LOADCNT 0
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr16_vgpr17, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAIT_LOADCNT 0
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr40_vgpr41, 0, 0, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16, $vgpr16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr8_vgpr9, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr16_vgpr17, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr40_vgpr41, 0, 0, implicit $exec
+...
>From 1c564216b841070dd54e00af8e1ce3eedff0ccce Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Thu, 7 May 2026 16:38:56 +0100
Subject: [PATCH 2/3] [AMDGPU] Increment VA_VDST twice for each VOP3PX2
instruction
In expert scheduling mode, change the VA_VDST counts to match the
hardware implementation. The inserted waits were conservatively correct
before. This just makes them more precise in some cases.
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 8 +++++++-
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 5 +++++
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 4 ++++
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 11 ++++++-----
.../test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir | 2 +-
5 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 2da41348244d7..b04e8a3159248 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1113,7 +1113,13 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
assert(T < Context->MaxCounter);
unsigned UB = getScoreUB(T);
- unsigned CurrScore = UB + 1;
+ unsigned Increment = 1;
+ if (T == AMDGPU::VA_VDST && AMDGPU::getHasMatrixScale(Inst.getOpcode())) {
+ // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
+ // two VOP3P instructions and increments VA_VDST twice.
+ Increment = 2;
+ }
+ unsigned CurrScore = UB + Increment;
if (CurrScore == 0)
report_fatal_error("InsertWaitcnt score wraparound");
// PendingEvents and ScoreUB need to be update regardless if this event
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 1c145359ccc61..b13aed2432602 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -588,6 +588,11 @@ bool getWMMAIsXDL(unsigned Opc) {
return Info ? Info->is_wmma_xdl : false;
}
+bool getHasMatrixScale(unsigned Opc) {
+ const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
+ return Info && Info->HasMatrixScale;
+}
+
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
switch (EncodingVal) {
case MFMAScaleFormats::FP6_E2M3:
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index f6b86a59b7b1d..49373f09ee460 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -126,6 +126,7 @@ struct True16D16Info {
struct WMMAInstInfo {
uint32_t Opcode;
bool is_wmma_xdl;
+ bool HasMatrixScale;
};
#define GET_MIMGBaseOpcode_DECL
@@ -618,6 +619,9 @@ bool getMAIIsGFX940XDL(unsigned Opc);
LLVM_READONLY
bool getWMMAIsXDL(unsigned Opc);
+LLVM_READONLY
+bool getHasMatrixScale(unsigned Opc);
+
// Get an equivalent BitOp3 for a binary logical \p Opc.
// \returns BitOp3 modifier for the logical operation or zero.
// Used in VOPD3 conversion.
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 7ae93b30e7f03..d95c9eb788700 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1752,15 +1752,16 @@ class VOP3PWMMA_Profile<list<ValueType> ArgTy, bit _IsSWMMAC, int _IndexType,
def WMMAInstInfoTable : GenericTable {
let FilterClass = "WMMAInstInfo";
let CppTypeName = "WMMAInstInfo";
- let Fields = ["Opcode", "is_wmma_xdl"];
+ let Fields = ["Opcode", "is_wmma_xdl", "HasMatrixScale"];
let PrimaryKey = ["Opcode"];
let PrimaryKeyName = "getWMMAInstInfoHelper";
}
-class WMMAInstInfo {
+class WMMAInstInfo<VOP3PWMMA_Profile WMMAProfile> {
Instruction Opcode = !cast<Instruction>(NAME);
bit is_wmma_xdl = 0;
+ bit HasMatrixScale = WMMAProfile.HasMatrixScale;
}
multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string PseudoInstrSuffix, bit DiffVdstSrc2 = 0> {
@@ -1770,14 +1771,14 @@ multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string Pse
let Mnemonic = Instr, mayRaiseFPException = 0, ReadsModeReg = 0, isConvergent = 1, SubtargetPredicate = HasWMMA128bInsts in {
let Constraints = WMMAConstraints2Addr, isConvertibleToThreeAddress = 1 in
- def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let PseudoInstr = Instr#PseudoInstrSuffix;
let FixedSize = WMMAProfile.HasMatrixScale;
let Size = !if(WMMAProfile.HasMatrixScale, 16, 8);
}
let Constraints = WMMAConstraints3Addr, SchedRW = [Write32Bit, Write32Bit] in
- def _threeaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _threeaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let PseudoInstr = Instr#PseudoInstrSuffix;
let FixedSize = WMMAProfile.HasMatrixScale;
let Size = !if(WMMAProfile.HasMatrixScale, 16, 8);
@@ -1789,7 +1790,7 @@ multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string Pse
}
multiclass SWMMACInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string PseudoInstrSuffix> {
- def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let Mnemonic = Instr;
let PseudoInstr = Instr#PseudoInstrSuffix;
let mayRaiseFPException = 0;
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
index 5647fef7d4302..f1d8a3305cd34 100644
--- a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
@@ -12,7 +12,7 @@ body: |
; CHECK-NEXT: early-clobber $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16, $vgpr16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
- ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_2
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_3
; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr8_vgpr9, 0, 0, implicit $exec
; CHECK-NEXT: S_WAIT_LOADCNT 0
; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
>From 788e1012872212ffae27fceb24175d7d473f9902 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at gmail.com>
Date: Fri, 8 May 2026 10:24:22 +0100
Subject: [PATCH 3/3] Update
llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
index f1d8a3305cd34..b149660c814a7 100644
--- a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs -amdgpu-expert-scheduling-mode -run-pass si-insert-waitcnts %s -o - -verify-machineinstrs | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-expert-scheduling-mode -run-pass=si-insert-waitcnts %s -o - | FileCheck %s
---
name: wmma_scale
More information about the llvm-commits
mailing list