[llvm-branch-commits] [llvm] [AMDGPU] Fix MFMA co-exec DS, VMEM, SALU and VALU stages for SMFMAC and DGEMM (PR #239095)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 4 21:55:13 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: lijinpei-amd
<details>
<summary>Changes</summary>
Depends on #<!-- -->239094.
getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after
these gfx950 MFMAs:
- SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL
16x16 entry, which allows VALU from cycle 2.
- SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4.
- DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0.
MI350X issues VALU at cycle 3 after both SMFMAC sizes.
The first commit adds DS, VMEM and SALU expectations to GFX950IssueStages
with the current values; the second updates them with the fix.
Assisted-by: Claude Opus 5.5
---
Full diff: https://github.com/llvm/llvm-project/pull/239095.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp (+12-6)
- (modified) llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp (+15-10)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 0e4441eeca2e3..3beab829f99c5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -136,9 +136,14 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
- // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
- // intended purposes here all those instructions are the same. This comment
- // is to simplify reverse mapping to the SPG.
+ Res.TotalWindow = 8;
+ allowCoExec(Res, CoExecMask::SALU, 1);
+ allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+ allowCoExec(Res, CoExecMask::WMMA, 4);
+ allowCoExec(Res, CoExecMask::PK_F32, 7);
+ return Res;
+
+ // SMFMAC, 4-cycle occupancy, 8-cycle window.
case V_SMFMAC_F32_16X16X64_BF16_e64:
case V_SMFMAC_F32_16X16X64_BF16_gfx940:
case V_SMFMAC_I32_16X16X128_I8_e64:
@@ -155,7 +160,8 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
case V_SMFMAC_F32_16X16X64_F16_gfx940:
Res.TotalWindow = 8;
allowCoExec(Res, CoExecMask::SALU, 1);
- allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+ allowCoExec(Res, CoExecMask::DS | CoExecMask::VMEM, 2);
+ allowCoExec(Res, CoExecMask::VALU, 3);
allowCoExec(Res, CoExecMask::WMMA, 4);
allowCoExec(Res, CoExecMask::PK_F32, 7);
return Res;
@@ -371,7 +377,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
case V_SMFMAC_F32_32X32X32_F16_gfx940:
Res.TotalWindow = 12;
allowCoExec(Res, CoExecMask::SALU, 1);
- allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 4);
+ allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 3);
allowCoExec(Res, CoExecMask::WMMA, 9);
allowCoExec(Res, CoExecMask::PK_F32, 11);
return Res;
@@ -382,7 +388,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
case V_MFMA_F64_16X16X4F64_vgprcd_e64:
Res.TotalWindow = 19;
- allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 0);
+ allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 1);
allowCoExec(Res, CoExecMask::WMMA | CoExecMask::VALU | CoExecMask::PK_F32,
18);
return Res;
diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index 48a9a359d392a..d67d5e0a2ecae 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -121,7 +121,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
// Opcode names without the _e64, _vgprcd_e64 or _mac*_e64 suffix.
std::vector<StringRef> MFMAs;
struct {
- unsigned VALU, PackedF32;
+ unsigned VALU, PackedF32, DS, VMEM, SALU;
} Expected;
} Groups[] = {
{{"V_MFMA_F32_16X16X32_BF16", "V_MFMA_F32_16X16X32_F16",
@@ -133,7 +133,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6",
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4",
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6"},
- {2, 7}},
+ {2, 7, 2, 2, 1}},
{{"V_MFMA_F32_16X16X128_F8F6F4_f4_f8",
"V_MFMA_F32_16X16X128_F8F6F4_f6_f8",
"V_MFMA_F32_16X16X128_F8F6F4_f8_f4",
@@ -144,7 +144,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4",
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6",
"V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8"},
- {3, 11}},
+ {3, 11, 2, 2, 1}},
{{"V_MFMA_F32_32X32X16_BF16", "V_MFMA_F32_32X32X16_F16",
"V_MFMA_I32_32X32X32_I8", "V_MFMA_F32_32X32X64_F8F6F4_f4_f4",
"V_MFMA_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_F32_32X32X64_F8F6F4_f6_f4",
@@ -153,7 +153,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6",
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4",
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6"},
- {2, 11}},
+ {2, 11, 2, 2, 1}},
{{"V_MFMA_F32_32X32X64_F8F6F4_f4_f8", "V_MFMA_F32_32X32X64_F8F6F4_f6_f8",
"V_MFMA_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_F32_32X32X64_F8F6F4_f8_f6",
"V_MFMA_F32_32X32X64_F8F6F4_f8_f8",
@@ -162,20 +162,18 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4",
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
"V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
- {3, 19}},
- // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
- // at 3.
+ {3, 19, 2, 2, 1}},
{{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
"V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
"V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
"V_SMFMAC_F32_16X16X128_FP8_FP8"},
- {2, 7}},
+ {3, 7, 2, 2, 1}},
{{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
"V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
"V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
"V_SMFMAC_F32_32X32X64_FP8_FP8"},
- {4, 11}},
- {{"V_MFMA_F64_16X16X4F64"}, {18, 18}},
+ {3, 11, 3, 3, 1}},
+ {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 1, 1, 1}},
};
for (const auto &G : Groups) {
for (StringRef Base : G.MFMAs) {
@@ -195,6 +193,13 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
EXPECT_EQ(Test.getIssueStage(MFMAOpc, V_PK_FMA_F32),
G.Expected.PackedF32)
<< Name;
+ CoExecInfo Info = getMFMACoExecInfo(MFMAOpc);
+ EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::DS), G.Expected.DS)
+ << Name;
+ EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::VMEM), G.Expected.VMEM)
+ << Name;
+ EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::SALU), G.Expected.SALU)
+ << Name;
}
EXPECT_GT(NumForms, 0u) << Base;
}
``````````
</details>
https://github.com/llvm/llvm-project/pull/239095
More information about the llvm-branch-commits
mailing list