[llvm] 4899e71 - [AMDGPU] Increment VA_VDST twice for each VOP3PX2 instruction (#196353)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 03:38:08 PDT 2026
Author: Jay Foad
Date: 2026-05-08T11:38:03+01:00
New Revision: 4899e71b0431a2c0ff5ca8745ed60b71f26da903
URL: https://github.com/llvm/llvm-project/commit/4899e71b0431a2c0ff5ca8745ed60b71f26da903
DIFF: https://github.com/llvm/llvm-project/commit/4899e71b0431a2c0ff5ca8745ed60b71f26da903.diff
LOG: [AMDGPU] Increment VA_VDST twice for each VOP3PX2 instruction (#196353)
In expert scheduling mode, change the VA_VDST counts to match the
hardware implementation. The inserted waits were conservatively correct
before. This just makes them more precise in some cases.
Added:
llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
Modified:
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
llvm/lib/Target/AMDGPU/VOP3PInstructions.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 12ba73cae0ce2..aaa01ee2e549a 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1113,7 +1113,13 @@ void WaitcntBrackets::updateByEvent(WaitEventType E, MachineInstr &Inst) {
assert(T < Context->MaxCounter);
unsigned UB = getScoreUB(T);
- unsigned CurrScore = UB + 1;
+ unsigned Increment = 1;
+ if (T == AMDGPU::VA_VDST && AMDGPU::getHasMatrixScale(Inst.getOpcode())) {
+ // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
+ // two VOP3P instructions and increments VA_VDST twice.
+ Increment = 2;
+ }
+ unsigned CurrScore = UB + Increment;
if (CurrScore == 0)
report_fatal_error("InsertWaitcnt score wraparound");
// PendingEvents and ScoreUB need to be update regardless if this event
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 1c145359ccc61..b13aed2432602 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -588,6 +588,11 @@ bool getWMMAIsXDL(unsigned Opc) {
return Info ? Info->is_wmma_xdl : false;
}
+bool getHasMatrixScale(unsigned Opc) {
+ const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
+ return Info && Info->HasMatrixScale;
+}
+
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
switch (EncodingVal) {
case MFMAScaleFormats::FP6_E2M3:
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index f6b86a59b7b1d..49373f09ee460 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -126,6 +126,7 @@ struct True16D16Info {
struct WMMAInstInfo {
uint32_t Opcode;
bool is_wmma_xdl;
+ bool HasMatrixScale;
};
#define GET_MIMGBaseOpcode_DECL
@@ -618,6 +619,9 @@ bool getMAIIsGFX940XDL(unsigned Opc);
LLVM_READONLY
bool getWMMAIsXDL(unsigned Opc);
+LLVM_READONLY
+bool getHasMatrixScale(unsigned Opc);
+
// Get an equivalent BitOp3 for a binary logical \p Opc.
// \returns BitOp3 modifier for the logical operation or zero.
// Used in VOPD3 conversion.
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 7ae93b30e7f03..d95c9eb788700 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1752,15 +1752,16 @@ class VOP3PWMMA_Profile<list<ValueType> ArgTy, bit _IsSWMMAC, int _IndexType,
def WMMAInstInfoTable : GenericTable {
let FilterClass = "WMMAInstInfo";
let CppTypeName = "WMMAInstInfo";
- let Fields = ["Opcode", "is_wmma_xdl"];
+ let Fields = ["Opcode", "is_wmma_xdl", "HasMatrixScale"];
let PrimaryKey = ["Opcode"];
let PrimaryKeyName = "getWMMAInstInfoHelper";
}
-class WMMAInstInfo {
+class WMMAInstInfo<VOP3PWMMA_Profile WMMAProfile> {
Instruction Opcode = !cast<Instruction>(NAME);
bit is_wmma_xdl = 0;
+ bit HasMatrixScale = WMMAProfile.HasMatrixScale;
}
multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string PseudoInstrSuffix, bit DiffVdstSrc2 = 0> {
@@ -1770,14 +1771,14 @@ multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string Pse
let Mnemonic = Instr, mayRaiseFPException = 0, ReadsModeReg = 0, isConvergent = 1, SubtargetPredicate = HasWMMA128bInsts in {
let Constraints = WMMAConstraints2Addr, isConvertibleToThreeAddress = 1 in
- def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let PseudoInstr = Instr#PseudoInstrSuffix;
let FixedSize = WMMAProfile.HasMatrixScale;
let Size = !if(WMMAProfile.HasMatrixScale, 16, 8);
}
let Constraints = WMMAConstraints3Addr, SchedRW = [Write32Bit, Write32Bit] in
- def _threeaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _threeaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let PseudoInstr = Instr#PseudoInstrSuffix;
let FixedSize = WMMAProfile.HasMatrixScale;
let Size = !if(WMMAProfile.HasMatrixScale, 16, 8);
@@ -1789,7 +1790,7 @@ multiclass WMMAInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string Pse
}
multiclass SWMMACInstGFX12<string Instr, VOP3PWMMA_Profile WMMAProfile, string PseudoInstrSuffix> {
- def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo {
+ def _twoaddr : VOP3P_Pseudo<Instr, WMMAProfile>, WMMAInstInfo<WMMAProfile> {
let Mnemonic = Instr;
let PseudoInstr = Instr#PseudoInstrSuffix;
let mayRaiseFPException = 0;
diff --git a/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
new file mode 100644
index 0000000000000..b149660c814a7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expert_scheduling_gfx1250.mir
@@ -0,0 +1,29 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-expert-scheduling-mode -run-pass=si-insert-waitcnts %s -o - | FileCheck %s
+
+---
+name: wmma_scale
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: wmma_scale
+ ; CHECK: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT 0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16, $vgpr16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_3
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr8_vgpr9, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAIT_LOADCNT 0
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_1
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr16_vgpr17, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAIT_LOADCNT 0
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; CHECK-NEXT: $vgpr64 = GLOBAL_LOAD_DWORD $vgpr40_vgpr41, 0, 0, implicit $exec
+ $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec
+ $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr16, $vgpr16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr8_vgpr9, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr16_vgpr17, 0, 0, implicit $exec
+ $vgpr64 = GLOBAL_LOAD_DWORD $vgpr40_vgpr41, 0, 0, implicit $exec
+...
More information about the llvm-commits
mailing list