[llvm-branch-commits] [llvm] [AMDGPU] Fix MFMA co-exec DS, VMEM, SALU and VALU stages for SMFMAC and DGEMM (PR #239095)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 00:23:54 PST 2026


https://github.com/lijinpei-amd updated https://github.com/llvm/llvm-project/pull/239095

>From 1e5e7b158260912a871e37117164b9bdd9ff9177 Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Thu, 5 Nov 2026 12:14:52 +0800
Subject: [PATCH 1/2] [AMDGPU] Precommit MFMA co-exec DS, VMEM and SALU stage
 tests (NFC)

Check the first cycle DS, VMEM and SALU can issue after each MFMA in
GFX950IssueStages. The expectations match the current table, which holds DS
and VMEM one cycle too long after SMFMAC 32x32 and allows DS, VMEM and SALU
one cycle too early after DGEMM 16x16x4.

Assisted-by: Claude Opus 5.5
---
 .../Target/AMDGPU/MFMACoExecRules.cpp         | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index 48a9a359d392ad..c7801421a704fc 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -121,7 +121,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
     // Opcode names without the _e64, _vgprcd_e64 or _mac*_e64 suffix.
     std::vector<StringRef> MFMAs;
     struct {
-      unsigned VALU, PackedF32;
+      unsigned VALU, PackedF32, DS, VMEM, SALU;
     } Expected;
   } Groups[] = {
       {{"V_MFMA_F32_16X16X32_BF16", "V_MFMA_F32_16X16X32_F16",
@@ -133,7 +133,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6"},
-       {2, 7}},
+       {2, 7, 2, 2, 1}},
       {{"V_MFMA_F32_16X16X128_F8F6F4_f4_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f6_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f8_f4",
@@ -144,7 +144,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8"},
-       {3, 11}},
+       {3, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X16_BF16", "V_MFMA_F32_32X32X16_F16",
         "V_MFMA_I32_32X32X32_I8", "V_MFMA_F32_32X32X64_F8F6F4_f4_f4",
         "V_MFMA_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_F32_32X32X64_F8F6F4_f6_f4",
@@ -153,7 +153,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6"},
-       {2, 11}},
+       {2, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X64_F8F6F4_f4_f8", "V_MFMA_F32_32X32X64_F8F6F4_f6_f8",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f8",
@@ -162,20 +162,21 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
-       {3, 19}},
+       {3, 19, 2, 2, 1}},
       // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
-      // at 3.
+      // at 3. It also holds DS and VMEM until 4 (32x32), not 3.
       {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
         "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
         "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
         "V_SMFMAC_F32_16X16X128_FP8_FP8"},
-       {2, 7}},
+       {2, 7, 2, 2, 1}},
       {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
         "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
         "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
         "V_SMFMAC_F32_32X32X64_FP8_FP8"},
-       {4, 11}},
-      {{"V_MFMA_F64_16X16X4F64"}, {18, 18}},
+       {4, 11, 4, 4, 1}},
+      // The table allows DS, VMEM and SALU at 0, not from 1.
+      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 0, 0, 0}},
   };
   for (const auto &G : Groups) {
     for (StringRef Base : G.MFMAs) {
@@ -195,6 +196,13 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         EXPECT_EQ(Test.getIssueStage(MFMAOpc, V_PK_FMA_F32),
                   G.Expected.PackedF32)
             << Name;
+        CoExecInfo Info = getMFMACoExecInfo(MFMAOpc);
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::DS), G.Expected.DS)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::VMEM), G.Expected.VMEM)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::SALU), G.Expected.SALU)
+            << Name;
       }
       EXPECT_GT(NumForms, 0u) << Base;
     }

>From 44c65327dc5a9aff400a58d84d67cd84ef033588 Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Thu, 5 Nov 2026 00:19:00 +0800
Subject: [PATCH 2/2] [AMDGPU] Fix MFMA co-exec DS, VMEM, SALU and VALU stages
 for SMFMAC and DGEMM

getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after
these gfx950 MFMAs:
- SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL
  16x16 entry, which allows VALU from cycle 2.
- SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4.
- DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0.

MI350X issues VALU at cycle 3 after both SMFMAC sizes.

Update the expected stages in GFX950IssueStages.

Assisted-by: Claude Opus 5.5
---
 .../AMDGPU/AMDGPUCoExecSchedStrategy.cpp       | 18 ++++++++++++------
 .../Target/AMDGPU/MFMACoExecRules.cpp          |  9 +++------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 7e4765a9422af9..6cecb5aef1bc06 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -122,9 +122,14 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
-    // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
-    // intended purposes here all those instructions are the same. This comment
-    // is to simplify reverse mapping to the SPG.
+    Res.TotalWindow = 8;
+    allowCoExec(Res, CoExecMask::SALU, 1);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::WMMA, 4);
+    allowCoExec(Res, CoExecMask::PK_F32, 7);
+    return Res;
+
+  // SMFMAC, 4-cycle occupancy, 8-cycle window.
   case V_SMFMAC_F32_16X16X64_BF16_e64:
   case V_SMFMAC_F32_16X16X64_BF16_gfx940:
   case V_SMFMAC_I32_16X16X128_I8_e64:
@@ -141,7 +146,8 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_16X16X64_F16_gfx940:
     Res.TotalWindow = 8;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::VALU, 3);
     allowCoExec(Res, CoExecMask::WMMA, 4);
     allowCoExec(Res, CoExecMask::PK_F32, 7);
     return Res;
@@ -357,7 +363,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_32X32X32_F16_gfx940:
     Res.TotalWindow = 12;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 4);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 3);
     allowCoExec(Res, CoExecMask::WMMA, 9);
     allowCoExec(Res, CoExecMask::PK_F32, 11);
     return Res;
@@ -368,7 +374,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
   case V_MFMA_F64_16X16X4F64_vgprcd_e64:
     Res.TotalWindow = 19;
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 0);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 1);
     allowCoExec(Res, CoExecMask::WMMA | CoExecMask::VALU | CoExecMask::PK_F32,
                 18);
     return Res;
diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index c7801421a704fc..d67d5e0a2ecae9 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -163,20 +163,17 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
        {3, 19, 2, 2, 1}},
-      // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
-      // at 3. It also holds DS and VMEM until 4 (32x32), not 3.
       {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
         "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
         "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
         "V_SMFMAC_F32_16X16X128_FP8_FP8"},
-       {2, 7, 2, 2, 1}},
+       {3, 7, 2, 2, 1}},
       {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
         "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
         "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
         "V_SMFMAC_F32_32X32X64_FP8_FP8"},
-       {4, 11, 4, 4, 1}},
-      // The table allows DS, VMEM and SALU at 0, not from 1.
-      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 0, 0, 0}},
+       {3, 11, 3, 3, 1}},
+      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 1, 1, 1}},
   };
   for (const auto &G : Groups) {
     for (StringRef Base : G.MFMAs) {



More information about the llvm-branch-commits mailing list