[llvm-branch-commits] [mlir] [mlir][AMDGPU] Add preferAgpr hint to MFMA operations (PR #242892)

Alan Li via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 15 20:34:03 PST 2026


https://github.com/lialan updated https://github.com/llvm/llvm-project/pull/242892

>From dad0d102e174a2df80d81c311c09a45d080bfe66 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Sun, 15 Nov 2026 14:02:45 -0800
Subject: [PATCH] [mlir][AMDGPU] Add preferAgpr hint to MFMA operations

The AMDGPU backend accepts the `!amdgpu.mfma !{!"agpr"}` instruction
metadata on calls to the `llvm.amdgcn.mfma.*` intrinsics (including
`llvm.amdgcn.mfma.scale.*`). The metadata is a hint that asks the backend
to allocate the accumulator input (C) and the result (D) of the MFMA to
AGPRs. The backend can ignore the hint, and dropping it does not change
program behavior.

Add an optional `preferAgpr` unit attribute to the ROCDL MFMA intrinsic
operations, including the scaled MFMA operations, and to `amdgpu.mfma` and
`amdgpu.scaled_mfma`. When the attribute is present, the translation to
LLVM IR attaches `!amdgpu.mfma !{!"agpr"}` to the intrinsic call, and the
AMDGPUToROCDL conversion copies the attribute to the ROCDL operation. When
the attribute is absent, the printed operations and the translated LLVM IR
are the same as before.

The SMFMAC operations do not get the attribute because the backend ignores
the hint on `llvm.amdgcn.smfmac.*`.

A single helper in the ROCDL translation builds the metadata, so a change
of the LLVM IR spelling of the hint is a change in one location.

Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
 .../mlir/Dialect/AMDGPU/IR/AMDGPUOps.td       | 27 +++++++--
 mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td  | 56 ++++++++++++++++---
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           |  4 ++
 .../ROCDL/ROCDLToLLVMIRTranslation.cpp        |  9 +++
 .../Conversion/AMDGPUToROCDL/mfma-gfx950.mlir | 14 +++++
 mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir  | 12 ++++
 mlir/test/Dialect/AMDGPU/ops.mlir             | 18 ++++++
 mlir/test/Dialect/LLVMIR/rocdl.mlir           | 32 +++++++++++
 mlir/test/Target/LLVMIR/rocdl.mlir            | 27 +++++++++
 9 files changed, 188 insertions(+), 11 deletions(-)

diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
index 398e754356ece8..bf7436bc2e5e36 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
@@ -1057,7 +1057,8 @@ def AMDGPU_MFMAOp :
                    UnitAttr:$reducePrecision,
                    UnitAttr:$negateA,
                    UnitAttr:$negateB,
-                   UnitAttr:$negateC)>,
+                   UnitAttr:$negateC,
+                   UnitAttr:$preferAgpr)>,
     Results<(outs MFMAOutTypes: $destD)> {
   let summary = "MLIR wrapper for CDNA mfma instructions";
   let description = [{
@@ -1087,6 +1088,12 @@ def AMDGPU_MFMAOp :
     The negateA, negateB, and negateC flags are only supported for double-precision
     operations on gfx94x.
 
+    The `preferAgpr` flag is a hint that asks the backend to allocate `destC`
+    and `destD` to AGPRs. The lowering to ROCDL copies the flag to the
+    intrinsic operation, and the translation to LLVM IR attaches the
+    `!amdgpu.mfma !{!"agpr"}` metadata to the intrinsic call. The backend can
+    ignore the hint, and the hint does not change the result.
+
     Example:
     ```mlir
       %0 = amdgpu.mfma 16x16x16 %matA * %matB + %matC
@@ -1095,6 +1102,9 @@ def AMDGPU_MFMAOp :
       %1 = amdgpu.mfma blocks(2) 32x32x1 %matD * %matE + %matF
         abid(1) blgp(bcast_second_32) cbsz(1)
         : f32, f32, vector<32xf32>
+
+      %2 = amdgpu.mfma 16x16x16 %matA * %matB + %matC preferAgpr
+        : vector<4xf16>, vector<4xf16>, vector<4xf32>
     ```
   }];
   let assemblyFormat = [{
@@ -1106,7 +1116,8 @@ def AMDGPU_MFMAOp :
           | `reducePrecision` $reducePrecision
           | `negateA` $negateA
           | `negateB` $negateB
-          | `negateC` $negateC)
+          | `negateC` $negateC
+          | `preferAgpr` $preferAgpr)
     attr-dict
     `:` type($sourceA) `,` type($sourceB) `,` type($destC)
   }];
@@ -1628,7 +1639,8 @@ def AMDGPU_ScaledMFMAOp :
                    AnyTypeOf<[F8E8M0FNU, FixedVectorOfLengthAndType<[4], [F8E8M0FNU]>]>:$scalesA,
                    AnyTypeOf<[F8E8M0FNU, FixedVectorOfLengthAndType<[4], [F8E8M0FNU]>]>:$scalesB,
                    ConfinedAttr<I32Attr, [IntNonNegative, IntMaxValue<3>]>:$scalesIdxA,
-                   ConfinedAttr<I32Attr, [IntNonNegative, IntMaxValue<3>]>:$scalesIdxB
+                   ConfinedAttr<I32Attr, [IntNonNegative, IntMaxValue<3>]>:$scalesIdxB,
+                   UnitAttr:$preferAgpr
                    )>,
     Results<(outs ScaledMFMAOutTypes: $destD)> {
   let summary = "MLIR wrapper for CDNA scaled mfma instructions";
@@ -1657,17 +1669,24 @@ def AMDGPU_ScaledMFMAOp :
     - The `negateA`, `negateB`, and `negateC` flags in `amdgpu.mfma` are only supported
       for double-precision operations on gfx94x and so are not included here.
 
+    The `preferAgpr` flag has the same meaning as in `amdgpu.mfma`: it is a
+    hint that asks the backend to allocate `destC` and `destD` to AGPRs. The
+    backend can ignore the hint, and the hint does not change the result.
+
     Example:
     ```mlir
       %0 = amdgpu.scaled_mfma 32x32x64 (%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %arg2
         : vector<4xf8E8M0FNU>, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
+
+      %1 = amdgpu.scaled_mfma 32x32x64 (%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %arg2 preferAgpr
+        : vector<4xf8E8M0FNU>, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
     ```
   }];
   let assemblyFormat = [{
     custom<MNKDimensionList>($m, $n, $k) ` `
     `(` $scalesA `[` $scalesIdxA `]` `*` $sourceA `)` `*`
     `(` $scalesB `[` $scalesIdxB `]` `*` $sourceB `)` `+` $destC
-    attr-dict
+    (`preferAgpr` $preferAgpr^)? attr-dict
     `:` type($scalesA) `,` type($sourceA) `,` type($scalesB) `,` type($sourceB) `,` type($destC)
   }];
   let hasCanonicalizer = 1;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index d8e458b97f18f1..5bed6855cec37c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -830,20 +830,44 @@ def ROCDL_IglpOpt : ROCDL_ConcreteNonMemIntrOp<"iglp.opt", [], 0, [0], ["variant
 //===---------------------------------------------------------------------===//
 // Xdlops intrinsics
 
+// Shared parts of the MFMA operations that have the `preferAgpr` attribute.
+class ROCDL_MfmaAgprHint {
+  // Translation code that attaches the AGPR hint to the intrinsic call when
+  // the `preferAgpr` attribute is present. `setMfmaAgprHint` is defined in
+  // ROCDLToLLVMIRTranslation.cpp.
+  string mfmaAgprHintLlvmBuilder = [{
+    if ($preferAgpr)
+      setMfmaAgprHint(inst);
+  }];
+
+  // Description text for the `preferAgpr` attribute.
+  string mfmaAgprHintDescription = [{
+    If the `preferAgpr` unit attribute is present, the call to the intrinsic
+    gets the `!amdgpu.mfma !{!"agpr"}` metadata. This metadata is a hint that
+    asks the backend to allocate the accumulator input (C) and the result (D) to
+    AGPRs. The backend can ignore the hint, and the hint does not change the
+    result.
+  }];
+}
+
 class ROCDL_Mfma_IntrOpBase<string mnemonic, ROCDL_NamedType ABType,
                             ROCDL_NamedType CDType, Attr blgpAttr> :
   ROCDL_IntrOp<mnemonic, [], [], [], 1, 0, 0, 0, [3, 4, 5], ["cbsz", "abid", "blgp"]>,
+  ROCDL_MfmaAgprHint,
   Arguments<(ins
              ABType:$a,
              ABType:$b,
              CDType:$c,
              I32Attr:$cbsz,
              I32Attr:$abid,
-             blgpAttr:$blgp)> {
+             blgpAttr:$blgp,
+             UnitAttr:$preferAgpr)> {
   let results = (outs CDType:$res);
+  let llvmBuilder = baseLlvmBuilder # mfmaAgprHintLlvmBuilder
+    # baseLlvmBuilderCoda;
   let assemblyFormat = [{
     $a `,` $b `,` $c `,` $cbsz `,` $abid `,` enum($blgp)
-    attr-dict `:` functional-type(operands, $res)
+    (`preferAgpr` $preferAgpr^)? attr-dict `:` functional-type(operands, $res)
   }];
 }
 
@@ -854,12 +878,17 @@ class ROCDL_Mfma_IntrOp<string mnemonic, ROCDL_NamedType ABType,
     Matrix fused multiply-add (MFMA) intrinsic. Computes `D = A * B + C`
     with matrix operands. The `cbsz`, `abid`, and `blgp` attributes control
     broadcast and block layout modes.
-
+  }] # mfmaAgprHintDescription # [{
     Example:
     ```mlir
     %r0 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, none : (}] # ABType.typeName
     # [{, }] # ABType.typeName # [{, }] # CDType.typeName # [{) -> }]
     # CDType.typeName # [{
+
+    // With the AGPR hint.
+    %r1 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, none preferAgpr : (}]
+    # ABType.typeName # [{, }] # ABType.typeName # [{, }] # CDType.typeName
+    # [{) -> }] # CDType.typeName # [{
     ```}];
 }
 
@@ -869,18 +898,24 @@ class ROCDL_Mfma_F64_IntrOp<string mnemonic, ROCDL_NamedType ABType,
   let description = [{
     Double-precision matrix fused multiply-add (MFMA) intrinsic. On gfx94x,
     the `blgp` immarg is a NEG bitfield rather than a B-lane permutation.
-
+  }] # mfmaAgprHintDescription # [{
     Example:
     ```mlir
     %r0 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, neg_a|neg_b : (}]
     # ABType.typeName # [{, }] # ABType.typeName # [{, }] # CDType.typeName
     # [{) -> }] # CDType.typeName # [{
+
+    // With the AGPR hint.
+    %r1 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, none preferAgpr : (}]
+    # ABType.typeName # [{, }] # ABType.typeName # [{, }] # CDType.typeName
+    # [{) -> }] # CDType.typeName # [{
     ```
   }];
 }
 
 class ROCDL_Mfma_Scale_IntrOp<string mnemonic, Type AB, Type CD> :
   ROCDL_IntrOp<mnemonic, [], [0, 1], [], 1, 0, 0, 0, [3, 4, 5, 7], ["cbsz", "blgp", "opselA", "opselB"]>,
+  ROCDL_MfmaAgprHint,
   Arguments<(ins
              LLVM_VectorOf<AB>:$a,
              LLVM_VectorOf<AB>:$b,
@@ -890,18 +925,21 @@ class ROCDL_Mfma_Scale_IntrOp<string mnemonic, Type AB, Type CD> :
              I32Attr:$opselA,
              I32:$scaleA,
              I32Attr:$opselB,
-             I32:$scaleB)> {
+             I32:$scaleB,
+             UnitAttr:$preferAgpr)> {
   let results = (outs LLVM_ScalarOrVectorOf<CD>:$res);
+  let llvmBuilder = baseLlvmBuilder # mfmaAgprHintLlvmBuilder
+    # baseLlvmBuilderCoda;
   let assemblyFormat = [{
     $a `,` $b `,` $c `,` enum($cbsz) `,` enum($blgp) `,` $opselA `,`
-    $scaleA `,` $opselB `,` $scaleB attr-dict `:`
+    $scaleA `,` $opselB `,` $scaleB (`preferAgpr` $preferAgpr^)? attr-dict `:`
     functional-type(operands, $res)
   }];
   let description = [{
     Scaled matrix fused multiply-add (MFMA) intrinsic with per-operand scaling.
     The `opselA`/`opselB` and `scaleA`/`scaleB` arguments control the scaling
     of input operands.
-
+  }] # mfmaAgprHintDescription # [{
     Example:
     ```mlir
     // Scaled MFMA with fp8 * fp8 inputs.
@@ -915,6 +953,10 @@ class ROCDL_Mfma_Scale_IntrOp<string mnemonic, Type AB, Type CD> :
     // Scaled MFMA with fp8 * fp6 inputs (6xi32 operand B).
     %r2 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %b6, %c, fp8_e4m3, fp6_e2m3, 0, %scaleA, 0, %scaleB :
       (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+
+    // Scaled MFMA with fp8 * fp8 inputs and the AGPR hint.
+    %r3 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %a, %c, fp8_e4m3, fp8_e4m3, 0, %scaleA, 0, %scaleB preferAgpr :
+      (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
     ```
   }];
 }
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 92a47dab42f019..58bf7f01968c25 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -1617,6 +1617,8 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
            {"abid", rewriter.getI32IntegerAttr(op.getAbid())},
            {"blgp", blgpAttr}});
     };
+    if (op.getPreferAgpr())
+      loweredOp.addAttribute("preferAgpr", rewriter.getUnitAttr());
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     if (outType != intrinsicOutType)
       lowered = LLVM::BitcastOp::create(rewriter, loc, outType, lowered);
@@ -1665,6 +1667,8 @@ struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
           ROCDL::MatrixFormatAttr::get(rewriter.getContext(), bTypeCode)},
          {"opselA", rewriter.getI32IntegerAttr(adaptor.getScalesIdxA())},
          {"opselB", rewriter.getI32IntegerAttr(adaptor.getScalesIdxB())}});
+    if (op.getPreferAgpr())
+      loweredOp.addAttribute("preferAgpr", rewriter.getUnitAttr());
 
     Value lowered = rewriter.create(loweredOp)->getResult(0);
     rewriter.replaceOp(op, lowered);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.cpp
index 303a27b879cecb..2ae581e6b8e95c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.cpp
@@ -27,6 +27,15 @@ using namespace mlir;
 using namespace mlir::LLVM;
 using mlir::LLVM::detail::createIntrinsicCall;
 
+/// Attaches the hint that asks the AMDGPU backend to allocate the accumulator
+/// input (C) and the result (D) of the MFMA intrinsic call `inst` to AGPRs.
+/// This is the only place that knows the LLVM IR spelling of the hint.
+static void setMfmaAgprHint(llvm::Instruction *inst) {
+  llvm::LLVMContext &ctx = inst->getContext();
+  inst->setMetadata("amdgpu.mfma",
+                    llvm::MDNode::get(ctx, llvm::MDString::get(ctx, "agpr")));
+}
+
 namespace {
 /// Implementation of the dialect interface that converts operations belonging
 /// to the ROCDL dialect to LLVM IR.
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
index 8e5bd123121bec..647676ac67d5d5 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
@@ -96,3 +96,17 @@ func.func @scaled_mfma_to_rocdl(%arg0 : vector<16xf32>,
 
   func.return
 }
+
+// CHECK-LABEL: func @mfma_prefer_agpr_to_rocdl
+func.func @mfma_prefer_agpr_to_rocdl(%arg0 : vector<16xf32>,
+                                     %arg1 : vector<32xf8E4M3FN>,
+                                     %arg2 : vector<4xf8E8M0FNU>,
+                                     %arg3 : f8E8M0FNU) {
+  // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %{{.*}}, 0, %{{.*}} preferAgpr : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  amdgpu.mfma 32x32x64 %arg1 * %arg1 + %arg0 preferAgpr : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<16xf32>
+  // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %{{.*}}, 1, %{{.*}} preferAgpr : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  amdgpu.scaled_mfma 32x32x64 (%arg2[0] * %arg1) * (%arg3[1] * %arg1) + %arg0 preferAgpr : vector<4xf8E8M0FNU>, vector<32xf8E4M3FN>, f8E8M0FNU, vector<32xf8E4M3FN>, vector<16xf32>
+  // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %{{.*}}, 1, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  amdgpu.scaled_mfma 32x32x64 (%arg2[0] * %arg1) * (%arg3[1] * %arg1) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf8E4M3FN>, f8E8M0FNU, vector<32xf8E4M3FN>, vector<16xf32>
+  func.return
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
index 2440851fc099f7..c36f0da9d66c75 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
@@ -95,3 +95,15 @@ func.func @mfma_to_rocdl(%arg0 : f32, %arg1 : vector<32xf32>,
 
   func.return
 }
+
+// CHECK-LABEL: func @mfma_prefer_agpr_to_rocdl
+func.func @mfma_prefer_agpr_to_rocdl(%arg0 : vector<4xf16>, %arg1 : vector<4xf32>,
+                                     %arg2 : f64) {
+  // CHECK: rocdl.mfma.f32.16x16x16f16 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, none preferAgpr : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  amdgpu.mfma 16x16x16 %arg0 * %arg0 + %arg1 preferAgpr : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  // CHECK: rocdl.mfma.f32.16x16x16f16 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, none : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  amdgpu.mfma 16x16x16 %arg0 * %arg0 + %arg1 : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  // CHECK: rocdl.mfma.f64.4x4x4f64 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, neg_a preferAgpr : (f64, f64, f64) -> f64
+  amdgpu.mfma blocks(4) 4x4x4 %arg2 * %arg2 + %arg2 negateA preferAgpr : f64, f64, f64
+  func.return
+}
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index 6477bb32ad2d15..7c5d8070062865 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -574,6 +574,15 @@ func.func @mfma_with_blocks(%arg0 : f32, %arg1 : vector<32xf32>) -> vector<32xf3
   func.return %0 : vector<32xf32>
 }
 
+// CHECK-LABEL: func @mfma_prefer_agpr
+func.func @mfma_prefer_agpr(%arg0 : vector<4xf16>, %arg1 : vector<4xf32>) -> vector<4xf32> {
+  // CHECK: amdgpu.mfma 16x16x16 %{{.*}} * %{{.*}} + %{{.*}} preferAgpr : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  %0 = amdgpu.mfma 16x16x16 %arg0 * %arg0 + %arg1 preferAgpr : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  // CHECK: amdgpu.mfma 16x16x16 %{{.*}} * %{{.*}} + %{{.*}} : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  %1 = amdgpu.mfma 16x16x16 %arg0 * %arg0 + %0 : vector<4xf16>, vector<4xf16>, vector<4xf32>
+  func.return %1 : vector<4xf32>
+}
+
 // CHECK-LABEL: func @wmma_f16_16x16x16_f16
 func.func @wmma_f16_16x16x16_f16(%arg0 : vector<16xf16>, %arg1 : vector<8xf16>) -> vector<8xf16> {
   // CHECK: amdgpu.wmma 16x16x16
@@ -651,6 +660,15 @@ func.func @scaled_mfma(%arg0 : f8E8M0FNU, %arg1 : vector<32xf6E2M3FN>, %arg2 : v
   func.return %0 : vector<16xf32>
 }
 
+// CHECK-LABEL: func @scaled_mfma_prefer_agpr
+func.func @scaled_mfma_prefer_agpr(%arg0 : f8E8M0FNU, %arg1 : vector<32xf6E2M3FN>, %arg2 : vector<16xf32>) -> vector<16xf32> {
+  // CHECK: amdgpu.scaled_mfma 32x32x64 (%{{.*}}[0] * %{{.*}}) * (%{{.*}}[1] * %{{.*}}) + %{{.*}} preferAgpr : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
+  %0 = amdgpu.scaled_mfma 32x32x64 (%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %arg2 preferAgpr : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
+  // CHECK: amdgpu.scaled_mfma 32x32x64 (%{{.*}}[0] * %{{.*}}) * (%{{.*}}[1] * %{{.*}}) + %{{.*}} : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
+  %1 = amdgpu.scaled_mfma 32x32x64 (%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %0 : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
+  func.return %1 : vector<16xf32>
+}
+
 // CHECK-LABEL: func @sparse_mfma
 func.func @sparse_mfma(%a16_4 : vector<4xf16>, %b16_8 : vector<8xf16>,
                        %a16_8 : vector<8xf16>, %b16_16 : vector<16xf16>,
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 66c69654d28734..0d0b239df58475 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -602,6 +602,38 @@ llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,
   llvm.return
 }
 
+llvm.func @rocdl.mfma.prefer_agpr(%a : vector<4xf16>, %c : vector<4xf32>,
+                                  %d : f64, %scale : i32,
+                                  %sa : vector<8xi32>,
+                                  %sc : vector<16xf32>) {
+  // CHECK-LABEL: rocdl.mfma.prefer_agpr
+  // CHECK: rocdl.mfma.f32.16x16x16f16 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, none preferAgpr : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  %r0 = rocdl.mfma.f32.16x16x16f16 %a, %a, %c, 0, 0, none preferAgpr :
+                            (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+
+  // CHECK: rocdl.mfma.f32.16x16x16f16 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, none : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  %r1 = rocdl.mfma.f32.16x16x16f16 %a, %a, %c, 0, 0, none :
+                            (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+
+  // CHECK: rocdl.mfma.f64.4x4x4f64 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, neg_a preferAgpr : (f64, f64, f64) -> f64
+  %r2 = rocdl.mfma.f64.4x4x4f64 %d, %d, %d, 0, 0, neg_a preferAgpr :
+                            (f64, f64, f64) -> f64
+
+  // CHECK: rocdl.mfma.f64.4x4x4f64 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, neg_a : (f64, f64, f64) -> f64
+  %r3 = rocdl.mfma.f64.4x4x4f64 %d, %d, %d, 0, 0, neg_a :
+                            (f64, f64, f64) -> f64
+
+  // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %{{.*}}, 0, %{{.*}} preferAgpr : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  %r4 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %sa, %sa, %sc, fp8_e4m3, fp8_e4m3, 0, %scale, 0, %scale preferAgpr :
+                            (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+
+  // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %{{.*}}, 0, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  %r5 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %sa, %sa, %sc, fp8_e4m3, fp8_e4m3, 0, %scale, 0, %scale :
+                            (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+
+  llvm.return
+}
+
 llvm.func @rocdl.mfma.scale.f32.16x16x128.f8f6f4(%arg0 : i32,
                    %arg1 : vector<4 x f32>, %arg2 : vector<8xi32>,
                    %arg3 : vector<6xi32>, %arg4 : vector<4xi32>) {
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index f8b21e0be13311..4485a6f5f22390 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -1992,6 +1992,32 @@ llvm.func @rocdl_last_use(%ptr: !llvm.ptr<1>) -> i32 {
   llvm.return %ret : i32
 }
 
+llvm.func @rocdl_mfma_prefer_agpr(%a : vector<4xf16>, %c : vector<4xf32>,
+                                  %d : f64, %scale : i32,
+                                  %sa : vector<8xi32>,
+                                  %sc : vector<16xf32>) {
+  // CHECK-LABEL: @rocdl_mfma_prefer_agpr
+  // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 0, i32 0), !amdgpu.mfma ![[$MFMA_AGPR:[0-9]+]]{{$}}
+  %r0 = rocdl.mfma.f32.16x16x16f16 %a, %a, %c, 0, 0, none preferAgpr :
+                            (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  // CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 0, i32 0){{$}}
+  %r1 = rocdl.mfma.f32.16x16x16f16 %a, %a, %c, 0, 0, none :
+                            (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
+  // CHECK: call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %{{.*}}, double %{{.*}}, double %{{.*}}, i32 0, i32 0, i32 1), !amdgpu.mfma ![[$MFMA_AGPR]]{{$}}
+  %r2 = rocdl.mfma.f64.4x4x4f64 %d, %d, %d, 0, 0, neg_a preferAgpr :
+                            (f64, f64, f64) -> f64
+  // CHECK: call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %{{.*}}, double %{{.*}}, double %{{.*}}, i32 0, i32 0, i32 1){{$}}
+  %r3 = rocdl.mfma.f64.4x4x4f64 %d, %d, %d, 0, 0, neg_a :
+                            (f64, f64, f64) -> f64
+  // CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}}), !amdgpu.mfma ![[$MFMA_AGPR]]{{$}}
+  %r4 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %sa, %sa, %sc, fp8_e4m3, fp8_e4m3, 0, %scale, 0, %scale preferAgpr :
+                            (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  // CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}}){{$}}
+  %r5 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %sa, %sa, %sc, fp8_e4m3, fp8_e4m3, 0, %scale, 0, %scale :
+                            (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+  llvm.return
+}
+
 llvm.func @test_fmed3_f16(%arg0: f16, %arg1: f16, %arg2: f16) -> f16 {
   // CHECK-LABEL: define half @test_fmed3_f16(half %0, half %1, half %2)
   %0 = rocdl.fmed3 %arg0, %arg1, %arg2 : f16
@@ -2262,6 +2288,7 @@ llvm.func @rocdl_dot_fp8_family(%i32: i32, %f32: f32) -> f32 {
 // CHECK-DAG: attributes #[[$KERNEL_NO_UNIFORM_WORK_GROUPS_ATTRS]] = { "amdgpu-flat-work-group-size"="1,256" }
 // CHECK-DAG: ![[$REQD_WORK_GROUP_SIZE]] = !{i32 16, i32 4, i32 2}
 // CHECK-DAG: ![[$REQD_BLOCK_SIZE]] = !{i32 8, i32 4, i32 2}
+// CHECK-DAG: ![[$MFMA_AGPR]] = !{!"agpr"}
 // CHECK-DAG: attributes #[[$KERNEL_WAVES_PER_EU_ATTR]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="2" "uniform-work-group-size" }
 // CHECK-DAG: attributes #[[$KERNEL_WAVES_PER_EU_RANGE_ATTR]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size" }
 // CHECK-DAG: attributes #[[$KERNEL_UNSAFE_FP_ATOMICS_ATTR]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-unsafe-fp-atomics"="true" "uniform-work-group-size" }



More information about the llvm-branch-commits mailing list