[llvm-branch-commits] [llvm] [AMDGPU] Fix MFMA co-exec DS, VMEM, SALU and VALU stages for SMFMAC and DGEMM (PR #239095)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 4 21:55:13 PST 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-amdgpu

Author: lijinpei-amd

<details>
<summary>Changes</summary>

Depends on #<!-- -->239094.

getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after
these gfx950 MFMAs:
- SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL
  16x16 entry, which allows VALU from cycle 2.
- SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4.
- DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0.

MI350X issues VALU at cycle 3 after both SMFMAC sizes.

The first commit adds DS, VMEM and SALU expectations to GFX950IssueStages
with the current values; the second updates them with the fix.

Assisted-by: Claude Opus 5.5

---
Full diff: https://github.com/llvm/llvm-project/pull/239095.diff


2 Files Affected:

- (modified) llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp (+12-6) 
- (modified) llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp (+15-10) 


``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 0e4441eeca2e3..3beab829f99c5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -136,9 +136,14 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
-    // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
-    // intended purposes here all those instructions are the same. This comment
-    // is to simplify reverse mapping to the SPG.
+    Res.TotalWindow = 8;
+    allowCoExec(Res, CoExecMask::SALU, 1);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::WMMA, 4);
+    allowCoExec(Res, CoExecMask::PK_F32, 7);
+    return Res;
+
+  // SMFMAC, 4-cycle occupancy, 8-cycle window.
   case V_SMFMAC_F32_16X16X64_BF16_e64:
   case V_SMFMAC_F32_16X16X64_BF16_gfx940:
   case V_SMFMAC_I32_16X16X128_I8_e64:
@@ -155,7 +160,8 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_16X16X64_F16_gfx940:
     Res.TotalWindow = 8;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::VALU, 3);
     allowCoExec(Res, CoExecMask::WMMA, 4);
     allowCoExec(Res, CoExecMask::PK_F32, 7);
     return Res;
@@ -371,7 +377,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_32X32X32_F16_gfx940:
     Res.TotalWindow = 12;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 4);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 3);
     allowCoExec(Res, CoExecMask::WMMA, 9);
     allowCoExec(Res, CoExecMask::PK_F32, 11);
     return Res;
@@ -382,7 +388,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
   case V_MFMA_F64_16X16X4F64_vgprcd_e64:
     Res.TotalWindow = 19;
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 0);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 1);
     allowCoExec(Res, CoExecMask::WMMA | CoExecMask::VALU | CoExecMask::PK_F32,
                 18);
     return Res;
diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index 48a9a359d392a..d67d5e0a2ecae 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -121,7 +121,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
     // Opcode names without the _e64, _vgprcd_e64 or _mac*_e64 suffix.
     std::vector<StringRef> MFMAs;
     struct {
-      unsigned VALU, PackedF32;
+      unsigned VALU, PackedF32, DS, VMEM, SALU;
     } Expected;
   } Groups[] = {
       {{"V_MFMA_F32_16X16X32_BF16", "V_MFMA_F32_16X16X32_F16",
@@ -133,7 +133,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6"},
-       {2, 7}},
+       {2, 7, 2, 2, 1}},
       {{"V_MFMA_F32_16X16X128_F8F6F4_f4_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f6_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f8_f4",
@@ -144,7 +144,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8"},
-       {3, 11}},
+       {3, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X16_BF16", "V_MFMA_F32_32X32X16_F16",
         "V_MFMA_I32_32X32X32_I8", "V_MFMA_F32_32X32X64_F8F6F4_f4_f4",
         "V_MFMA_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_F32_32X32X64_F8F6F4_f6_f4",
@@ -153,7 +153,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6"},
-       {2, 11}},
+       {2, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X64_F8F6F4_f4_f8", "V_MFMA_F32_32X32X64_F8F6F4_f6_f8",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f8",
@@ -162,20 +162,18 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
-       {3, 19}},
-      // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
-      // at 3.
+       {3, 19, 2, 2, 1}},
       {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
         "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
         "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
         "V_SMFMAC_F32_16X16X128_FP8_FP8"},
-       {2, 7}},
+       {3, 7, 2, 2, 1}},
       {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
         "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
         "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
         "V_SMFMAC_F32_32X32X64_FP8_FP8"},
-       {4, 11}},
-      {{"V_MFMA_F64_16X16X4F64"}, {18, 18}},
+       {3, 11, 3, 3, 1}},
+      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 1, 1, 1}},
   };
   for (const auto &G : Groups) {
     for (StringRef Base : G.MFMAs) {
@@ -195,6 +193,13 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         EXPECT_EQ(Test.getIssueStage(MFMAOpc, V_PK_FMA_F32),
                   G.Expected.PackedF32)
             << Name;
+        CoExecInfo Info = getMFMACoExecInfo(MFMAOpc);
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::DS), G.Expected.DS)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::VMEM), G.Expected.VMEM)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::SALU), G.Expected.SALU)
+            << Name;
       }
       EXPECT_GT(NumForms, 0u) << Base;
     }

``````````

</details>


https://github.com/llvm/llvm-project/pull/239095


More information about the llvm-branch-commits mailing list