[Mlir-commits] [mlir] 681fc74 - [mlir][ROCDL] Move ROCDL intrinsic enum immargs to enums (#198875)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Fri Jun 5 09:52:48 PDT 2026
Author: Krzysztof Drewniak
Date: 2026-06-05T09:52:44-07:00
New Revision: 681fc74ac47eaa597d22506231a347748dda635b
URL: https://github.com/llvm/llvm-project/commit/681fc74ac47eaa597d22506231a347748dda635b
DIFF: https://github.com/llvm/llvm-project/commit/681fc74ac47eaa597d22506231a347748dda635b.diff
LOG: [mlir][ROCDL] Move ROCDL intrinsic enum immargs to enums (#198875)
In many cases, a "i32" `immarg` arguhment to an intrinsic in the AMDGPU
backend actually corresponds directly to some enumerated set of values
in the backend, which we have to smuggle through an I32. This makes the
MLIR forms of intrinsics less readable and means that people either have
to use the `amdgpu` dialect to get these enums or have to roll their own
enums if they want to know what's going on.
This PR rips the band-aid off and breaks the world by swapping out those
integer attributes for enum attributes.
Of special note is the handling of the aux/cachepolicy field on various
intrinsics; in the backend, all the architectures share an enum and
you've just got to use the right names in the right spots. Here, we've
separated out the cases for pre-gfx942, gfx942+, and gfx12 enums as
separate attributes (including separate casing for gfx12 atomics) and
allowed any of them to be used. We also allow an I32Attr in those
arguments for easy importing and to make the common case of "0" portably
printable.
In all other cases, we've just moved to the new enums.
The WMMA ops, which have multiple immadiate arguments that could be
confusing, have been given assembly formats than name the filed names.
This commit also moves the AMDGPU dialct to use the new enums.
In order to be able to use the current `EnumAttr` system, a method has
been added to LLVM translation to allow such enum attributes to be
converted to their underlying integer constants.
AI usage: this is a quasi-mechanical refactor that was pretty much
farmed out to Codex
---------
Co-authored-by: OpenAI Codex <codex at openai.com>
Added:
mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.td
mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
mlir/test/Dialect/LLVMIR/rocdl-cache-policy-invalid.mlir
Modified:
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUAttrs.td
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUBase.td
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h
mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
mlir/include/mlir/Target/LLVM/ROCDL/Utils.h
mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
mlir/lib/Dialect/AMDGPU/IR/AMDGPUOps.cpp
mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
mlir/test/Dialect/AMDGPU/ops.mlir
mlir/test/Dialect/LLVMIR/rocdl.mlir
mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
mlir/test/Target/LLVMIR/rocdl.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUAttrs.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUAttrs.td
index 51e1267571243..65a5f044bf4d8 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUAttrs.td
@@ -11,6 +11,7 @@
include "mlir/Dialect/AMDGPU/IR/AMDGPUBase.td"
include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td"
+include "mlir/Dialect/LLVMIR/ROCDLAttrs.td"
def AMDGPU_AddressSpaceAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_AddressSpace,
"address_space"> {
@@ -39,14 +40,6 @@ def AMDGPU_AddressSpaceAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_AddressSpace,
def AMDGPU_DPPPermAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_DPPPerm,
"dpp_perm">;
-def AMDGPU_SchedBarrierOpOptAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_SchedBarrierOpOpt,
- "sched_barrier_opt">{
- let assemblyFormat = "`<` $value `>`";
-}
-
-def AMDGPU_MFMAPermBAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_MFMAPermB,
- "mfma_perm_b">;
-
def AMDGPU_LoadTemporalHintAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_LoadTemporalHint,
"load_temporal_hint">;
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUBase.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUBase.td
index 0ceea9334fc03..639dbf6b4a128 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUBase.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUBase.td
@@ -81,6 +81,9 @@ def AMDGPU_Dialect : Dialect {
in a more programmer-friendly fashion, such as through enums or though
using separate arguments that are later combined. (For example, see the
design of the `amdgpu.dpp` and `amdgpu.fat_raw_buffer_cast` operations.)
+ However, in the case where an immediate argument corresponds directly to an
+ enum in the backend compiler, that enum should be in the ROCDL dialect
+ instead.
If sufficiently similar functionality on multiple hardware generations can be
encapsulated into a single operation, it should be done. The lowering to
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
index 6ab91a0314c41..42d4a4e62f5e2 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
@@ -15,6 +15,7 @@
#define MLIR_DIALECT_AMDGPU_IR_AMDGPUDIALECT_H_
#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Dialect.h"
#include "mlir/IR/OpDefinition.h"
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td
index fe4723b635dc6..96bff45267af1 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.td
@@ -46,40 +46,6 @@ def AMDGPU_DPPPerm : I32Enum<"DPPPerm",
let cppNamespace = "::mlir::amdgpu";
}
-def AMDGPU_SchedBarrierOpOpt : I32BitEnum<"sched_barrier_opt_enum",
- "The possible options for scheduling barriers",
- [
- I32BitEnumAttrCaseNone<"none">,
- I32BitEnumAttrCaseBit<"non_mem_non_sideffect", 0>,
- I32BitEnumAttrCaseBit<"valu", 1>,
- I32BitEnumAttrCaseBit<"salu", 2>,
- I32BitEnumAttrCaseBit<"mfma_wmma", 3>,
- I32BitEnumAttrCaseBit<"all_vmem", 4>,
- I32BitEnumAttrCaseBit<"vmem_read", 5>,
- I32BitEnumAttrCaseBit<"vmem_write", 6>,
- I32BitEnumAttrCaseBit<"all_ds", 7>,
- I32BitEnumAttrCaseBit<"ds_read", 8>,
- I32BitEnumAttrCaseBit<"ds_write", 9>,
- I32BitEnumAttrCaseBit<"transcendental", 10>
- ]> {
- let cppNamespace = "::mlir::amdgpu";
-}
-
-def AMDGPU_MFMAPermB : I32Enum<"MFMAPermB",
- "The possible permutations of the lanes storing B available in an MFMA",
- [
- I32EnumAttrCase<"none", 0>,
- I32EnumAttrCase<"bcast_first_32", 1>,
- I32EnumAttrCase<"bcast_second_32", 2>,
- I32EnumAttrCase<"rotate_16_right", 3>,
- I32EnumAttrCase<"bcast_first_16", 4>,
- I32EnumAttrCase<"bcast_second_16", 5>,
- I32EnumAttrCase<"bcast_third_16", 6>,
- I32EnumAttrCase<"bcast_fourth_16", 7>
- ]> {
- let cppNamespace = "::mlir::amdgpu";
-}
-
def AMDGPU_LoadTemporalHint : I32Enum<"LoadTemporalHint",
"AMDGPU-specific prefetch temporal hints for load instructions. "
"RT - regular temporal for both near and far caches; "
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
index 8a15d3a76d17e..69a9adf3be8af 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUOps.td
@@ -910,13 +910,13 @@ def AMDGPU_LDSBarrierOp : AMDGPU_Op<"lds_barrier"> {
def AMDGPU_SchedBarrierOp :
AMDGPU_Op<"sched_barrier">,
- Arguments<(ins AMDGPU_SchedBarrierOpOptAttr:$opts)>
+ Arguments<(ins ROCDL_SchedGroupMaskAttr:$opts)>
{
let summary = "Barrier that limits the backend scheduler of instruction movement";
let description = [{
`amdgpu.sched_barrier` serves as a barrier that could be
configured to restrict movements of instructions through it as
- defined by sched_barrier_opts.
+ defined by the ROCDL scheduling group mask enum.
}];
let assemblyFormat = [{
`allow` `=` $opts attr-dict
@@ -1027,8 +1027,8 @@ def AMDGPU_MFMAOp :
MFMAOutTypes:$destC,
DefaultValuedAttr<I32Attr, "0">:$cbsz,
DefaultValuedAttr<I32Attr, "0">:$abid,
- DefaultValuedAttr<AMDGPU_MFMAPermBAttr,
- "::mlir::amdgpu::MFMAPermB::none">:$blgp,
+ DefaultValuedAttr<ROCDL_MFMAPermBAttr,
+ "::mlir::ROCDL::MFMAPermB::none">:$blgp,
UnitAttr:$reducePrecision,
UnitAttr:$negateA,
UnitAttr:$negateB,
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index 12fca3001ae69..de9a3e1c93282 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -78,6 +78,8 @@ add_mlir_dialect(ROCDLOps rocdl)
add_mlir_doc(ROCDLOps ROCDLDialect Dialects/ -gen-dialect-doc -dialect=rocdl)
set(LLVM_TARGET_DEFINITIONS ROCDLOps.td)
mlir_tablegen(ROCDLConversions.inc -gen-llvmir-conversions)
+mlir_tablegen(ROCDLOpsEnums.h.inc -gen-enum-decls)
+mlir_tablegen(ROCDLOpsEnums.cpp.inc -gen-enum-defs)
mlir_tablegen(ROCDLOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=rocdl)
mlir_tablegen(ROCDLOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=rocdl)
add_mlir_dialect_tablegen_target(MLIRROCDLConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h
index d3e5408b73764..e0e98e167f501 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h
@@ -15,6 +15,8 @@
#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
+#include <type_traits>
+
namespace mlir {
class LLVMTypeConverter;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index e05fb6a9bac7d..8c6d4fcefa903 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -551,6 +551,37 @@ def LLVM_LLVMAddrSpaceAttrInterface :
];
}
+def LLVM_IntrinsicIntegerAttrInterface :
+ AttrInterface<"IntrinsicIntegerAttrInterface"> {
+ let description = [{
+ An interface for attributes that represent LLVM intrinsic integer immediate
+ arguments while using a richer dialect-specific assembly form.
+
+ The default implementation is intended for enum-style attributes whose
+ `getValue()` returns an enum with a storage type that matches the LLVM
+ immarg width. Attributes with non-enum storage or a
diff erent LLVM immarg
+ width must override `getIntegerAttr`.
+ }];
+ let cppNamespace = "::mlir::LLVM";
+ let methods = [
+ InterfaceMethod<
+ /*description=*/"Returns the integer attribute to pass to LLVM.",
+ /*retTy=*/"::mlir::IntegerAttr",
+ /*methodName=*/"getIntegerAttr",
+ /*args=*/(ins),
+ /*methodBody=*/[{}],
+ /*defaultImplementation=*/[{
+ using EnumType = decltype($_attr.getValue());
+ using StorageType = std::underlying_type_t<EnumType>;
+ auto type = ::mlir::IntegerType::get($_attr.getContext(),
+ sizeof(StorageType) * 8);
+ return ::mlir::IntegerAttr::get(
+ type, static_cast<StorageType>($_attr.getValue()));
+ }]
+ >
+ ];
+}
+
def LLVM_TargetAttrInterface
: AttrInterface<"TargetAttrInterface", [DLTIQueryInterface]> {
let description = [{
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
new file mode 100644
index 0000000000000..c3a66e2e92256
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLAttrs.td
@@ -0,0 +1,100 @@
+//===-- ROCDLAttrs.td - ROCDL IR dialect attributes -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ROCDLIR_ATTRS
+#define ROCDLIR_ATTRS
+
+include "mlir/Dialect/LLVMIR/ROCDLDialect.td"
+include "mlir/Dialect/LLVMIR/ROCDLEnums.td"
+include "mlir/IR/EnumAttr.td"
+
+//===----------------------------------------------------------------------===//
+// ROCDL attribute definitions
+//===----------------------------------------------------------------------===//
+
+class ROCDL_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
+ : AttrDef<ROCDL_Dialect, attrName, traits> {
+ let mnemonic = attrMnemonic;
+}
+
+class ROCDL_IntrinsicIntegerEnumAttr<EnumInfo enumInfo, string name>
+ : EnumAttr<ROCDL_Dialect, enumInfo, name,
+ [LLVM_IntrinsicIntegerAttrInterface]>;
+
+def ROCDL_MFMAPermBAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_MFMAPermB, "mfma_perm_b">;
+
+def ROCDL_MFMANegModifierAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_MFMANegModifier,
+ "mfma_neg_modifier">;
+
+def ROCDL_MatrixFormatAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_MatrixFormat, "matrix_format">;
+
+def ROCDL_WMMAMatrixScaleAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_WMMAMatrixScale,
+ "wmma_matrix_scale">;
+
+def ROCDL_WMMAMatrixScaleFormatAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_WMMAMatrixScaleFormat,
+ "wmma_matrix_scale_format">;
+
+def ROCDL_WMMACModifierAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_WMMACModifier,
+ "wmma_c_modifier">;
+
+def ROCDL_SchedGroupMaskAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_SchedGroupMask,
+ "sched_group_mask">;
+
+def ROCDL_PreGfx12CachePolicyAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_PreGfx12CachePolicy,
+ "pre_gfx12_cache_policy">;
+
+def ROCDL_Gfx942CachePolicyAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_Gfx942CachePolicy,
+ "gfx942_cache_policy">;
+
+def ROCDL_Gfx12CachePolicyAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_Gfx12CachePolicy,
+ "gfx12_cache_policy">;
+
+def ROCDL_Gfx12AtomicCachePolicyAttr
+ : ROCDL_IntrinsicIntegerEnumAttr<ROCDL_Gfx12AtomicCachePolicy,
+ "gfx12_atomic_cache_policy">;
+
+class ROCDL_CachePolicyAttrOf<list<Attr> policyAttrs, string summary>
+ : AnyAttrOf<!listconcat(policyAttrs, [I32Attr]), summary> {
+ let storageType = "::mlir::Attribute";
+ let returnType = "::mlir::Attribute";
+ let convertFromStorage = "$_self";
+ let constBuilderCall = "$0";
+}
+
+class ROCDL_DefaultCachePolicyAttr<Attr policyAttr>
+ : DefaultValuedAttr<policyAttr, "$_builder.getI32IntegerAttr(0)">;
+
+def ROCDL_PreGfx12CachePolicyCompatAttr : ROCDL_CachePolicyAttrOf<[
+ ROCDL_PreGfx12CachePolicyAttr, ROCDL_Gfx942CachePolicyAttr],
+ "pre-gfx12 or gfx942 AMDGPU cache policy attribute">;
+
+def ROCDL_Gfx12NonAtomicCachePolicyCompatAttr : ROCDL_CachePolicyAttrOf<[
+ ROCDL_Gfx12CachePolicyAttr],
+ "gfx12 non-atomic AMDGPU cache policy attribute">;
+
+def ROCDL_NonAtomicBufferCachePolicyCompatAttr : ROCDL_CachePolicyAttrOf<[
+ ROCDL_PreGfx12CachePolicyAttr, ROCDL_Gfx942CachePolicyAttr,
+ ROCDL_Gfx12CachePolicyAttr],
+ "non-atomic AMDGPU buffer cache policy attribute">;
+
+def ROCDL_AtomicBufferCachePolicyCompatAttr : ROCDL_CachePolicyAttrOf<[
+ ROCDL_PreGfx12CachePolicyAttr, ROCDL_Gfx942CachePolicyAttr,
+ ROCDL_Gfx12AtomicCachePolicyAttr],
+ "atomic AMDGPU buffer cache policy attribute">;
+
+#endif // ROCDLIR_ATTRS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h b/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
index ce1fe5a03c494..58d35a4373656 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.h
@@ -28,6 +28,8 @@
#include "mlir/IR/OpDefinition.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Dialect/LLVMIR/ROCDLOpsEnums.h.inc"
+
///// Ops /////
#define GET_ATTRDEF_CLASSES
#include "mlir/Dialect/LLVMIR/ROCDLOpsAttributes.h.inc"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.td
new file mode 100644
index 0000000000000..6b3f073a4a994
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLDialect.td
@@ -0,0 +1,155 @@
+//===-- ROCDLDialect.td - ROCDL IR dialect definition -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ROCDLIR_DIALECT
+#define ROCDLIR_DIALECT
+
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+
+//===----------------------------------------------------------------------===//
+// ROCDL dialect definitions
+//===----------------------------------------------------------------------===//
+
+def ROCDL_Dialect : Dialect {
+ let name = "rocdl";
+ let cppNamespace = "::mlir::ROCDL";
+ let dependentDialects = ["LLVM::LLVMDialect"];
+ let summary = "Dialect for wrapping LLVM AMDGPU backend intrinsics and attributes";
+ let hasOperationAttrVerify = 1;
+
+ let description = [{
+ The ROCDL dialect, like the other platform-specific LLVM dialects, serves
+ as the location of wrappers around the AMD-specific intrinsics and attributes
+ in LLVM.
+
+ This dialect, like other GPU lowering targets, also contains the infrastructure
+ used by the built-in compilation/offloading framework to compile AMD-specific
+ LLVM IR into binaries.
+
+ # Dialect inclusion criteria and guidelines
+
+ The operations in this dialect are 1:1 wrappers around their corresponding
+ LLVM intrinsics. Operations that do not correspond to intrinsics should not
+ be placed in this dialect.
+
+ The definition of a ROCDL op should match its LLVM counterpart. If the
+ argument and result types are fixed, they should be specified as type
+ constraints, including by overriding the default variadic type on LLVM
+ intrinsics by doing a `let results` in the operation definition.
+
+ As an exception, if an `immarg` intrinsic argument corresponds to a backend
+ enum, it should be represented in MLIR by an attribute that models that
+ enum.
+
+ LLVM attributes do not need to be replicated exactly if it wouldn't be
+ easy to do so, but pure operations and ones that read/write memory should
+ be annotated as such.
+
+ While LLVM intrinsics currently don't allow constraining the values an
+ `any_type` can take, it is acceptable (but not required) to impose such
+ constraints if they are known.
+
+ When an LLVM intrinsic uses an `immarg`, this corresponds to an attribute
+ in MLIR.
+
+ Human-readable assembly formats (those that, for example, explicitly indicate
+ parameter names) may be used, and are encouraged for intrinsics that have
+ complex argument schemes and don't have any higher-level wrapper (such as
+ in the `amdgpu` dialect).
+
+ While not all existing operations follow this convention, new operations should
+ generally provide argument and result types except in cases where they are
+ clearly redundant (such as with operations like `rocdl.fmed3`, which doesn't
+ need to reiterate the single type at issue multiple times). This convention
+ enhances the readability of low-level IR and prevents programmers from needing
+ to find non-local type information.
+
+ Dialect-defined discardable attributes (any attribute starting with `rocdl.`
+ that has special handling) need to correspond to AMD-specific attributes, metadata,
+ or other entities (such as calling conventions) in LLVM, or be needed for
+ GPU compilation management. Outside of the compilation infrastructure,
+ dialect-specific enums or attributes are extremely unlikely to be needed
+ and should be avoided.
+
+ Operation documentation should specify when the operation was introduced
+ (if relevant) and include usage examples. Operations should have
+ parser/printer tests in `mlir/test/Dialect/LLVMIR/rocdl.mlir` and
+ lowering tests in `mlir/test/Target/LLVMIR/rocdl.mlir`.
+
+ # General documentation (What does this op do?)
+
+ While rocdl ops sometimes carry their own documentation, there is no
+ expectation that such documentation will exist (or be kept up to date).
+
+ Since ROCDL operations correspond to LLVM intrinsics, the semantics and
+ behavior of these operations can be determined by investigating the
+ documentation for the corresponding intrinsic. This documentation
+ can be found in
+ - `llvm/docs/AMDGPUUsage.rst` and
+ - The comments of `llvm/include/llvm/IR/IntrinsicsAMDGPU.td`, which
+ is where details of the meaning of certain bitfields or of how an
+ intrinsic corresponds to hardware instructions are most likely to
+ be found.
+
+ Since many intrinsics are themselves minimal wrappers around hardware
+ instructions, these documentation sources often do not repeat hardware
+ documentation. If an intrinsic appears undocumented, information about
+ its behavior will often be available in published ISA descriptions or
+ (sometimes known as shader programming guides).
+
+ If an operation doesn't provide usage examples, it is likely that they
+ can be found in `mlir/test/Dialect/LLVMIR/rocdl.mlir` (op syntax and
+ verification) or `mlir/test/Target/LLVMIR/rocdl.mlir` (translation
+ to LLVM IR).
+ }];
+
+ let extraClassDeclaration = [{
+ /// Get the name of the attribute used to annotate external kernel
+ /// functions.
+ static StringRef getKernelFuncAttrName() { return "rocdl.kernel"; }
+ static constexpr ::llvm::StringLiteral getFlatWorkGroupSizeAttrName() {
+ return ::llvm::StringLiteral("rocdl.flat_work_group_size");
+ }
+ static constexpr ::llvm::StringLiteral getReqdWorkGroupSizeAttrName() {
+ return ::llvm::StringLiteral("rocdl.reqd_work_group_size");
+ }
+ /// MLIR's gpu-related infrastructure effectively assume uniform workgroup
+ /// sizes, so this attribute defaults to "true" on `rocdl.kernel` functions.
+ /// It is provided here to allow overriding this assumption.
+ static constexpr ::llvm::StringLiteral getUniformWorkGroupSizeAttrName() {
+ return ::llvm::StringLiteral("rocdl.uniform_work_group_size");
+ }
+
+ /// The address space value that represents global memory.
+ static constexpr unsigned kGlobalMemoryAddressSpace = 1;
+ /// The address space value that represents shared memory.
+ static constexpr unsigned kSharedMemoryAddressSpace = 3;
+ /// The address space value that represents constant memory.
+ static constexpr unsigned kConstantMemoryAddressSpace = 4;
+ /// The address space value that represents private memory.
+ static constexpr unsigned kPrivateMemoryAddressSpace = 5;
+ }];
+
+ let discardableAttrs = (ins
+ "::mlir::UnitAttr":$kernel,
+ "::mlir::DenseI32ArrayAttr":$reqd_work_group_size,
+ "::mlir::StringAttr":$flat_work_group_size,
+ "::mlir::IntegerAttr":$max_flat_work_group_size,
+ "::mlir::IntegerAttr":$waves_per_eu,
+ "::mlir::BoolAttr":$unsafe_fp_atomics,
+ // Correspond to LLVM metadata of the same name
+ "::mlir::UnitAttr":$last_use,
+ "::mlir::UnitAttr":$no_remote_memory,
+ "::mlir::UnitAttr":$no_fine_grained_memory,
+ "::mlir::UnitAttr":$ignore_denormal_mode
+ );
+
+ let useDefaultAttributePrinterParser = 1;
+}
+
+#endif // ROCDLIR_DIALECT
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
new file mode 100644
index 0000000000000..e0d1773b59b7b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLEnums.td
@@ -0,0 +1,282 @@
+//===-- ROCDLEnums.td - ROCDL IR dialect enums ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ROCDLIR_ENUMS
+#define ROCDLIR_ENUMS
+
+include "mlir/IR/EnumAttr.td"
+
+//===----------------------------------------------------------------------===//
+// ROCDL enum definitions
+//===----------------------------------------------------------------------===//
+
+class ROCDL_I32Enum<string name, string summary, list<EnumCase> cases>
+ : I32Enum<name, summary, cases> {
+ let cppNamespace = "::mlir::ROCDL";
+}
+
+class ROCDL_I16BitEnum<string name, string summary,
+ list<BitEnumCaseBase> cases>
+ : I16BitEnum<name, summary, cases> {
+ let cppNamespace = "::mlir::ROCDL";
+}
+
+class ROCDL_I32BitEnum<string name, string summary,
+ list<BitEnumCaseBase> cases>
+ : I32BitEnum<name, summary, cases> {
+ let cppNamespace = "::mlir::ROCDL";
+}
+
+def ROCDL_MFMAPermB : ROCDL_I32Enum<"MFMAPermB",
+ "permutations of the lanes storing B in an MFMA",
+ [
+ I32EnumCase<"none", 0>,
+ I32EnumCase<"bcast_first_32", 1>,
+ I32EnumCase<"bcast_second_32", 2>,
+ I32EnumCase<"rotate_16_right", 3>,
+ I32EnumCase<"bcast_first_16", 4>,
+ I32EnumCase<"bcast_second_16", 5>,
+ I32EnumCase<"bcast_third_16", 6>,
+ I32EnumCase<"bcast_fourth_16", 7>
+ ]>;
+
+def ROCDL_MFMANegModifierNone : I32BitEnumCaseNone<"none">;
+def ROCDL_MFMANegModifierA : I32BitEnumCaseBit<"neg_a", 0>;
+def ROCDL_MFMANegModifierB : I32BitEnumCaseBit<"neg_b", 1>;
+def ROCDL_MFMANegModifierC : I32BitEnumCaseBit<"neg_c", 2>;
+def ROCDL_MFMANegModifier : ROCDL_I32BitEnum<"MFMANegModifier",
+ "negation modifier bitfield for gfx94x double-precision MFMA",
+ [
+ ROCDL_MFMANegModifierNone,
+ ROCDL_MFMANegModifierA,
+ ROCDL_MFMANegModifierB,
+ ROCDL_MFMANegModifierC
+ ]> {
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_MatrixFormat : ROCDL_I32Enum<"MatrixFormat",
+ "matrix operand formats selected by scaled MFMA/WMMA format fields",
+ [
+ I32EnumCase<"fp8_e4m3", 0>,
+ I32EnumCase<"fp8_e5m2", 1>,
+ I32EnumCase<"fp6_e2m3", 2>,
+ I32EnumCase<"fp6_e3m2", 3>,
+ I32EnumCase<"fp4_e2m1", 4>
+ ]>;
+
+def ROCDL_WMMAMatrixScale : ROCDL_I32Enum<"WMMAMatrixScale",
+ "matrix scale row selector",
+ [
+ I32EnumCase<"row0", 0>,
+ I32EnumCase<"row1", 1>
+ ]>;
+
+def ROCDL_WMMAMatrixScaleFormat : ROCDL_I32Enum<"WMMAMatrixScaleFormat",
+ "matrix scale exponent formats",
+ [
+ I32EnumCase<"e8", 0>,
+ I32EnumCase<"e5m3", 1>,
+ I32EnumCase<"e4m3", 2>
+ ]>;
+
+def ROCDL_WMMACModifierNone : I16BitEnumCaseNone<"none">;
+def ROCDL_WMMACModifierNeg : I16BitEnumCaseBit<"neg", 0>;
+def ROCDL_WMMACModifierAbs : I16BitEnumCaseBit<"abs", 1>;
+def ROCDL_WMMACModifierNegAbs : BitEnumCaseGroup<"neg_abs",
+ [ROCDL_WMMACModifierNeg, ROCDL_WMMACModifierAbs]>;
+def ROCDL_WMMACModifier : ROCDL_I16BitEnum<"WMMACModifier",
+ "WMMA C operand modifiers",
+ [
+ ROCDL_WMMACModifierNone,
+ ROCDL_WMMACModifierNeg,
+ ROCDL_WMMACModifierAbs,
+ ROCDL_WMMACModifierNegAbs
+ ]> {
+ let printBitEnumPrimaryGroups = 1;
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_SchedGroupMaskNone : I32BitEnumCaseNone<"none">;
+def ROCDL_SchedGroupMaskNonMemNonSideEffect
+ : I32BitEnumCaseBit<"non_mem_non_sideeffect", 0>;
+def ROCDL_SchedGroupMaskVALU : I32BitEnumCaseBit<"valu", 1>;
+def ROCDL_SchedGroupMaskSALU : I32BitEnumCaseBit<"salu", 2>;
+def ROCDL_SchedGroupMaskMFMAWMMA : I32BitEnumCaseBit<"mfma_wmma", 3>;
+def ROCDL_SchedGroupMaskAllVMEM : I32BitEnumCaseBit<"all_vmem", 4>;
+def ROCDL_SchedGroupMaskVMEMRead : I32BitEnumCaseBit<"vmem_read", 5>;
+def ROCDL_SchedGroupMaskVMEMWrite : I32BitEnumCaseBit<"vmem_write", 6>;
+def ROCDL_SchedGroupMaskAllDS : I32BitEnumCaseBit<"all_ds", 7>;
+def ROCDL_SchedGroupMaskDSRead : I32BitEnumCaseBit<"ds_read", 8>;
+def ROCDL_SchedGroupMaskDSWrite : I32BitEnumCaseBit<"ds_write", 9>;
+def ROCDL_SchedGroupMaskTranscendental
+ : I32BitEnumCaseBit<"transcendental", 10>;
+def ROCDL_SchedGroupMaskLDSDMA : I32BitEnumCaseBit<"ldsdma", 11>;
+def ROCDL_SchedGroupMaskAll : BitEnumCaseGroup<"all",
+ [ROCDL_SchedGroupMaskNonMemNonSideEffect, ROCDL_SchedGroupMaskVALU,
+ ROCDL_SchedGroupMaskSALU, ROCDL_SchedGroupMaskMFMAWMMA,
+ ROCDL_SchedGroupMaskAllVMEM, ROCDL_SchedGroupMaskVMEMRead,
+ ROCDL_SchedGroupMaskVMEMWrite, ROCDL_SchedGroupMaskAllDS,
+ ROCDL_SchedGroupMaskDSRead, ROCDL_SchedGroupMaskDSWrite,
+ ROCDL_SchedGroupMaskTranscendental, ROCDL_SchedGroupMaskLDSDMA]>;
+def ROCDL_SchedGroupMask : ROCDL_I32BitEnum<"SchedGroupMask",
+ "instruction type mask for scheduling barriers",
+ [
+ ROCDL_SchedGroupMaskNone,
+ ROCDL_SchedGroupMaskNonMemNonSideEffect,
+ ROCDL_SchedGroupMaskVALU,
+ ROCDL_SchedGroupMaskSALU,
+ ROCDL_SchedGroupMaskMFMAWMMA,
+ ROCDL_SchedGroupMaskAllVMEM,
+ ROCDL_SchedGroupMaskVMEMRead,
+ ROCDL_SchedGroupMaskVMEMWrite,
+ ROCDL_SchedGroupMaskAllDS,
+ ROCDL_SchedGroupMaskDSRead,
+ ROCDL_SchedGroupMaskDSWrite,
+ ROCDL_SchedGroupMaskTranscendental,
+ ROCDL_SchedGroupMaskLDSDMA,
+ ROCDL_SchedGroupMaskAll
+ ]> {
+ let printBitEnumPrimaryGroups = 1;
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_PreGfx12CachePolicyNone : I32BitEnumCaseNone<"none">;
+def ROCDL_PreGfx12CachePolicyGLC : I32BitEnumCaseBit<"glc", 0>;
+def ROCDL_PreGfx12CachePolicySLC : I32BitEnumCaseBit<"slc", 1>;
+def ROCDL_PreGfx12CachePolicyDLC : I32BitEnumCaseBit<"dlc", 2>;
+def ROCDL_PreGfx12CachePolicySWZ : I32BitEnumCaseBit<"swz", 3>;
+def ROCDL_PreGfx12CachePolicySCC : I32BitEnumCaseBit<"scc", 4>;
+def ROCDL_PreGfx12CachePolicyAll : BitEnumCaseGroup<"all",
+ [ROCDL_PreGfx12CachePolicyGLC, ROCDL_PreGfx12CachePolicySLC,
+ ROCDL_PreGfx12CachePolicyDLC, ROCDL_PreGfx12CachePolicySCC]>;
+def ROCDL_PreGfx12CachePolicyVolatile
+ : I32BitEnumCaseBit<"volatile_op", 31, "volatile">;
+def ROCDL_PreGfx12CachePolicy : ROCDL_I32BitEnum<"PreGfx12CachePolicy",
+ "pre-gfx12 buffer cache policy bits",
+ [
+ ROCDL_PreGfx12CachePolicyNone,
+ ROCDL_PreGfx12CachePolicyGLC,
+ ROCDL_PreGfx12CachePolicySLC,
+ ROCDL_PreGfx12CachePolicyDLC,
+ ROCDL_PreGfx12CachePolicySWZ,
+ ROCDL_PreGfx12CachePolicySCC,
+ ROCDL_PreGfx12CachePolicyAll,
+ ROCDL_PreGfx12CachePolicyVolatile
+ ]> {
+ let printBitEnumPrimaryGroups = 1;
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_Gfx942CachePolicyNone : I32BitEnumCaseNone<"none">;
+def ROCDL_Gfx942CachePolicySC0 : I32BitEnumCaseBit<"sc0", 0>;
+def ROCDL_Gfx942CachePolicyNT : I32BitEnumCaseBit<"nt", 1>;
+def ROCDL_Gfx942CachePolicySWZ : I32BitEnumCaseBit<"swz", 3>;
+def ROCDL_Gfx942CachePolicySC1 : I32BitEnumCaseBit<"sc1", 4>;
+def ROCDL_Gfx942CachePolicyVolatile
+ : I32BitEnumCaseBit<"volatile_op", 31, "volatile">;
+def ROCDL_Gfx942CachePolicy : ROCDL_I32BitEnum<"Gfx942CachePolicy",
+ "gfx942 buffer cache policy bits",
+ [
+ ROCDL_Gfx942CachePolicyNone,
+ ROCDL_Gfx942CachePolicySC0,
+ ROCDL_Gfx942CachePolicyNT,
+ ROCDL_Gfx942CachePolicySWZ,
+ ROCDL_Gfx942CachePolicySC1,
+ ROCDL_Gfx942CachePolicyVolatile
+ ]> {
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_Gfx12CachePolicyNone : I32BitEnumCaseNone<"none">;
+def ROCDL_Gfx12CachePolicyNT : I32BitEnumCaseBit<"nt", 0>;
+def ROCDL_Gfx12CachePolicyHT : I32BitEnumCaseBit<"ht", 1>;
+// Helper bit for temporal encodings. Keep this out of the enum case list: if
+// listed there, the generated printer may print this bit directly instead of
+// using the named nt_rt/rt_nt/nt_ht/nt_wb groups below.
+def ROCDL_Gfx12CachePolicyTemporalBit2 : I32BitEnumCaseBit<"temporal_bit2", 2>;
+def ROCDL_Gfx12CachePolicyLU : BitEnumCaseGroup<"lu",
+ [ROCDL_Gfx12CachePolicyNT, ROCDL_Gfx12CachePolicyHT]>;
+def ROCDL_Gfx12CachePolicyNTRT : BitEnumCaseGroup<"nt_rt",
+ [ROCDL_Gfx12CachePolicyTemporalBit2]>;
+def ROCDL_Gfx12CachePolicyRTNT : BitEnumCaseGroup<"rt_nt",
+ [ROCDL_Gfx12CachePolicyNT, ROCDL_Gfx12CachePolicyTemporalBit2]>;
+def ROCDL_Gfx12CachePolicyNTHT : BitEnumCaseGroup<"nt_ht",
+ [ROCDL_Gfx12CachePolicyHT, ROCDL_Gfx12CachePolicyTemporalBit2]>;
+def ROCDL_Gfx12CachePolicyNTWB : BitEnumCaseGroup<"nt_wb",
+ [ROCDL_Gfx12CachePolicyNT, ROCDL_Gfx12CachePolicyHT,
+ ROCDL_Gfx12CachePolicyTemporalBit2]>;
+def ROCDL_Gfx12CachePolicyScopeSE : I32BitEnumCaseBit<"scope_se", 3>;
+def ROCDL_Gfx12CachePolicyScopeDev : I32BitEnumCaseBit<"scope_dev", 4>;
+def ROCDL_Gfx12CachePolicyScopeSys : BitEnumCaseGroup<"scope_sys",
+ [ROCDL_Gfx12CachePolicyScopeSE, ROCDL_Gfx12CachePolicyScopeDev]>;
+def ROCDL_Gfx12CachePolicyNV : I32BitEnumCaseBit<"nv", 5>;
+def ROCDL_Gfx12CachePolicySWZ : I32BitEnumCaseBit<"swz", 6>;
+def ROCDL_Gfx12CachePolicySCAL : I32BitEnumCaseBit<"scal", 11>;
+def ROCDL_Gfx12CachePolicyVolatile
+ : I32BitEnumCaseBit<"volatile_op", 31, "volatile">;
+def ROCDL_Gfx12CachePolicy : ROCDL_I32BitEnum<"Gfx12CachePolicy",
+ "gfx12 cache policy bits",
+ [
+ ROCDL_Gfx12CachePolicyNone,
+ ROCDL_Gfx12CachePolicyNT,
+ ROCDL_Gfx12CachePolicyHT,
+ ROCDL_Gfx12CachePolicyLU,
+ ROCDL_Gfx12CachePolicyNTRT,
+ ROCDL_Gfx12CachePolicyRTNT,
+ ROCDL_Gfx12CachePolicyNTHT,
+ ROCDL_Gfx12CachePolicyNTWB,
+ ROCDL_Gfx12CachePolicyScopeSE,
+ ROCDL_Gfx12CachePolicyScopeDev,
+ ROCDL_Gfx12CachePolicyScopeSys,
+ ROCDL_Gfx12CachePolicyNV,
+ ROCDL_Gfx12CachePolicySWZ,
+ ROCDL_Gfx12CachePolicySCAL,
+ ROCDL_Gfx12CachePolicyVolatile
+ ]> {
+ let printBitEnumPrimaryGroups = 1;
+ let printBitEnumQuoted = 0;
+}
+
+def ROCDL_Gfx12AtomicCachePolicyNone : I32BitEnumCaseNone<"none">;
+// Note: this bit **will** be set automatically by the backend, you do
+// **not** need to include it in the policy, it's included for readability.
+def ROCDL_Gfx12AtomicCachePolicyReturn
+ : I32BitEnumCaseBit<"keep_return", 0, "return">;
+def ROCDL_Gfx12AtomicCachePolicyNT : I32BitEnumCaseBit<"nt", 1>;
+def ROCDL_Gfx12AtomicCachePolicyCascade
+ : I32BitEnumCaseBit<"cascade", 2>;
+def ROCDL_Gfx12AtomicCachePolicyScopeSE
+ : I32BitEnumCaseBit<"scope_se", 3>;
+def ROCDL_Gfx12AtomicCachePolicyScopeDev
+ : I32BitEnumCaseBit<"scope_dev", 4>;
+def ROCDL_Gfx12AtomicCachePolicyScopeSys
+ : BitEnumCaseGroup<"scope_sys",
+ [ROCDL_Gfx12AtomicCachePolicyScopeSE,
+ ROCDL_Gfx12AtomicCachePolicyScopeDev]>;
+def ROCDL_Gfx12AtomicCachePolicyNV : I32BitEnumCaseBit<"nv", 5>;
+def ROCDL_Gfx12AtomicCachePolicyVolatile
+ : I32BitEnumCaseBit<"volatile_op", 31, "volatile">;
+def ROCDL_Gfx12AtomicCachePolicy : ROCDL_I32BitEnum<"Gfx12AtomicCachePolicy",
+ "gfx12 atomic cache policy bits",
+ [
+ ROCDL_Gfx12AtomicCachePolicyNone,
+ ROCDL_Gfx12AtomicCachePolicyReturn,
+ ROCDL_Gfx12AtomicCachePolicyNT,
+ ROCDL_Gfx12AtomicCachePolicyCascade,
+ ROCDL_Gfx12AtomicCachePolicyScopeSE,
+ ROCDL_Gfx12AtomicCachePolicyScopeDev,
+ ROCDL_Gfx12AtomicCachePolicyScopeSys,
+ ROCDL_Gfx12AtomicCachePolicyNV,
+ ROCDL_Gfx12AtomicCachePolicyVolatile
+ ]> {
+ let printBitEnumPrimaryGroups = 1;
+ let printBitEnumQuoted = 0;
+}
+
+#endif // ROCDLIR_ENUMS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index b3b0407f8ed8f..ba185dc27dc7e 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -14,156 +14,10 @@
#define ROCDLIR_OPS
include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
+include "mlir/Dialect/LLVMIR/ROCDLAttrs.td"
include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
include "mlir/Interfaces/SideEffectInterfaces.td"
-//===----------------------------------------------------------------------===//
-// ROCDL dialect definitions
-//===----------------------------------------------------------------------===//
-
-def ROCDL_Dialect : Dialect {
- let name = "rocdl";
- let cppNamespace = "::mlir::ROCDL";
- let dependentDialects = ["LLVM::LLVMDialect"];
- let summary = "Dialect for wrapping LLVM AMDGPU backend intrinsics and attributes";
- let hasOperationAttrVerify = 1;
-
- let description = [{
- The ROCDL dialect, like the other platform-specific LLVM dialects, serves
- as the location of wrappers around the AMD-specific intrinsics and attributes
- in LLVM.
-
- This dialect, like other GPU lowering targets, also contains the infrastructure
- used by the built-in compilation/offloading framework to compile AMD-specific
- LLVM IR into binaries.
-
- # Dialect inclusion criteria and guidelines
-
- The operations in this dialect are 1:1 wrappers around their corresponding
- LLVM intrinsics. Operations that do not correspond to intrinsics should not
- be placed in this dialect.
-
- The definition of a ROCDL op should match its LLVM counterpart. If the
- argument and result types are fixed, they should be specified as type
- constraints, including by overriding the default variadic type on LLVM
- intrinsics by doing a `let results` in the operation definition.
-
- LLVM attributes do not need to be replicated exactly if it wouldn't be
- easy to do so, but pure operations and ones that read/write memory should
- be annotated as such.
-
- While LLVM intrinsics currently don't allow constraining the values an
- `any_type` can take, it is acceptable (but not required) to impose such
- constraints if they are known.
-
- When an LLVM intrinsic uses an `immarg`, this corresponds to an attribute
- in MLIR.
-
- Human-readable assembly formats (those that, for example, explicitly indicate
- parameter names) may be used, and are encouraged for intrinsics that have
- complex argument schemes and don't have any higher-level wrapper (such as
- in the `amdgpu` dialect).
-
- While not all existing operations follow this convention, new operations should
- generally provide argument and result types except in cases where they are
- clearly redundant (such as with operations like `rocdl.fmed3`, which doesn't
- need to reiterate the single type at issue multiple times). This convention
- enhances the readability of low-level IR and prevents programmers from needing
- to find non-local type information.
-
- Dialect-defined discardable attributes (any attribute starting with `rocdl.`
- that has special handling) need to correspond to AMD-specific attributes, metadata,
- or other entities (such as calling conventions) in LLVM, or be needed for
- GPU compilation management. Outside of the compilation infrastructure,
- dialect-specific enums or attributes are extmelely unlikely to be needed
- and should be avoided.
-
- Operation documentation should specify when the operation was introduced
- (if relevant) and include usage examples. Operations should have
- parser/printer tests in `mlir/test/Dialect/LLVMIR/rocdl.mlir` and
- lowering tests in `mlir/test/Target/LLVMIR/rocdl.mlir`.
-
- # General documentation (What does this op do?)
-
- While rocdl ops sometimes carry their own documentation, there is no
- expectation that such documentation will exist (or be kept up to date).
-
- Since ROCDL operations correspond to LLVM intrinsics, the semantics and
- behavior of these operations can be determined by investigating the
- documentation for the corresponding intrinsic. This documentation
- can be found in
- - `llvm/docs/AMDGPUUsage.rst` and
- - The comments of `llvm/include/llvm/IR/IntrinsicsAMDGPU.td`, which
- is where details of the meaning of certain bitfields or of how an
- intrinsic corresponds to hardware instructions are most likely to
- be found.
-
- Since many intrinsics are themselves minimal wrappers around hardware
- instructions, these documentation sources often do not repeat hardware
- documentation. If an intrinsic appears undocumented, information about
- its behavior will often be available in published ISA descriptions or
- (sometimes known as shader programming guides).
-
- If an operation doesn't provide usage examples, it is likely that they
- can be found in `mlir/test/Dialect/LLVMIR/rocdl.mlir` (op syntax and
- verification) or `mlir/test/Target/LLVMIR/rocdl.mlir` (translation
- to LLVM IR).
- }];
-
- let extraClassDeclaration = [{
- /// Get the name of the attribute used to annotate external kernel
- /// functions.
- static StringRef getKernelFuncAttrName() { return "rocdl.kernel"; }
- static constexpr ::llvm::StringLiteral getFlatWorkGroupSizeAttrName() {
- return ::llvm::StringLiteral("rocdl.flat_work_group_size");
- }
- static constexpr ::llvm::StringLiteral getReqdWorkGroupSizeAttrName() {
- return ::llvm::StringLiteral("rocdl.reqd_work_group_size");
- }
- /// MLIR's gpu-related infrastructure effectively assume uniform workgroup
- /// sizes, so this attribute defaults to "true" on `rocdl.kernel` functions.
- /// It is provided here to allow overriding this assumption.
- static constexpr ::llvm::StringLiteral getUniformWorkGroupSizeAttrName() {
- return ::llvm::StringLiteral("rocdl.uniform_work_group_size");
- }
-
- /// The address space value that represents global memory.
- static constexpr unsigned kGlobalMemoryAddressSpace = 1;
- /// The address space value that represents shared memory.
- static constexpr unsigned kSharedMemoryAddressSpace = 3;
- /// The address space value that represents constant memory.
- static constexpr unsigned kConstantMemoryAddressSpace = 4;
- /// The address space value that represents private memory.
- static constexpr unsigned kPrivateMemoryAddressSpace = 5;
- }];
-
- let discardableAttrs = (ins
- "::mlir::UnitAttr":$kernel,
- "::mlir::DenseI32ArrayAttr":$reqd_work_group_size,
- "::mlir::StringAttr":$flat_work_group_size,
- "::mlir::IntegerAttr":$max_flat_work_group_size,
- "::mlir::IntegerAttr":$waves_per_eu,
- "::mlir::BoolAttr":$unsafe_fp_atomics,
- // Correspond to LLVM metadata of the same name
- "::mlir::UnitAttr":$last_use,
- "::mlir::UnitAttr":$no_remote_memory,
- "::mlir::UnitAttr":$no_fine_grained_memory,
- "::mlir::UnitAttr":$ignore_denormal_mode
- );
-
- let useDefaultAttributePrinterParser = 1;
-}
-
-//===----------------------------------------------------------------------===//
-// ROCDL attribute definitions
-//===----------------------------------------------------------------------===//
-
-class ROCDL_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
- : AttrDef<ROCDL_Dialect, attrName, traits> {
- let mnemonic = attrMnemonic;
-}
-
-
//===----------------------------------------------------------------------===//
// ROCDL op definitions
//===----------------------------------------------------------------------===//
@@ -902,20 +756,22 @@ def ROCDL_SetPrioOp : ROCDL_ConcreteNonMemIntrOp<"s.setprio", [], 0, [0], ["prio
}
def ROCDL_SchedBarrier : ROCDL_ConcreteNonMemIntrOp<"sched.barrier", [], 0, [0],["mask"]>,
- Arguments<(ins I32Attr:$mask)> {
+ Arguments<(ins ROCDL_SchedGroupMaskAttr:$mask)> {
let assemblyFormat = "$mask attr-dict";
let description = [{
Insert a scheduling barrier with the given mask. The mask is a
bitfield that controls which instruction types may be scheduled
- across the barrier (e.g. `0x0000` = no instructions may cross,
- `0x0001` = ALU only, `0x0010` = all VMEM, etc.). See
- https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/IntrinsicsAMDGPU.td#L349
- for the full list of mask values.
+ across the barrier. The mask values mirror the `llvm.amdgcn.sched.barrier`
+ intrinsic's documented mask values and the AMDGPU backend's
+ `SchedGroupMask` enum.
Example:
```mlir
- // Scheduling barrier with mask 0.
- rocdl.sched.barrier 0
+ // Scheduling barrier with no instructions allowed to cross.
+ rocdl.sched.barrier none
+
+ // Allow VALU and all VMEM instructions to cross.
+ rocdl.sched.barrier valu|all_vmem
```
}];
}
@@ -923,15 +779,17 @@ def ROCDL_SchedBarrier : ROCDL_ConcreteNonMemIntrOp<"sched.barrier", [], 0, [0],
def ROCDL_SchedGroupBarrier
: ROCDL_ConcreteNonMemIntrOp<"sched.group.barrier", [], 0,
[0, 1, 2], ["mask", "size", "groupId"]>,
- Arguments<(ins I32Attr:$mask, I32Attr:$size, I32Attr:$groupId)> {
+ Arguments<(ins ROCDL_SchedGroupMaskAttr:$mask, I32Attr:$size,
+ I32Attr:$groupId)> {
let assemblyFormat = "$mask `,` $size `,` $groupId attr-dict";
let description = [{
- Insert a scheduling group barrier.
+ Insert a scheduling group barrier. The first parameter uses the same
+ scheduling group mask values as `rocdl.sched.barrier`.
Example:
```mlir
// Schedule group barrier with mask, size, and group id.
- rocdl.sched.group.barrier 8, 1, 0
+ rocdl.sched.group.barrier mfma_wmma, 1, 0
```
}];
}
@@ -953,7 +811,8 @@ def ROCDL_IglpOpt : ROCDL_ConcreteNonMemIntrOp<"iglp.opt", [], 0, [0], ["variant
//===---------------------------------------------------------------------===//
// Xdlops intrinsics
-class ROCDL_Mfma_IntrOp<string mnemonic, ROCDL_NamedType ABType, ROCDL_NamedType CDType> :
+class ROCDL_Mfma_IntrOpBase<string mnemonic, ROCDL_NamedType ABType,
+ ROCDL_NamedType CDType, Attr blgpAttr> :
ROCDL_IntrOp<mnemonic, [], [], [], 1, 0, 0, 0, [3, 4, 5], ["cbsz", "abid", "blgp"]>,
Arguments<(ins
ABType:$a,
@@ -961,11 +820,16 @@ class ROCDL_Mfma_IntrOp<string mnemonic, ROCDL_NamedType ABType, ROCDL_NamedType
CDType:$c,
I32Attr:$cbsz,
I32Attr:$abid,
- I32Attr:$blgp)> {
+ blgpAttr:$blgp)> {
let results = (outs CDType:$res);
let assemblyFormat = [{
$a `,` $b `,` $c `,` $cbsz `,` $abid `,` $blgp attr-dict `:` functional-type(operands, $res)
}];
+}
+
+class ROCDL_Mfma_IntrOp<string mnemonic, ROCDL_NamedType ABType,
+ ROCDL_NamedType CDType> :
+ ROCDL_Mfma_IntrOpBase<mnemonic, ABType, CDType, ROCDL_MFMAPermBAttr> {
let description = [{
Matrix fused multiply-add (MFMA) intrinsic. Computes `D = A * B + C`
with matrix operands. The `cbsz`, `abid`, and `blgp` attributes control
@@ -973,20 +837,36 @@ class ROCDL_Mfma_IntrOp<string mnemonic, ROCDL_NamedType ABType, ROCDL_NamedType
Example:
```mlir
- %r0 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, 0 : (}] # ABType.typeName
+ %r0 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, none : (}] # ABType.typeName
# [{, }] # ABType.typeName # [{, }] # CDType.typeName # [{) -> }]
# CDType.typeName # [{
```}];
}
+class ROCDL_Mfma_F64_IntrOp<string mnemonic, ROCDL_NamedType ABType,
+ ROCDL_NamedType CDType> :
+ ROCDL_Mfma_IntrOpBase<mnemonic, ABType, CDType, ROCDL_MFMANegModifierAttr> {
+ let description = [{
+ Double-precision matrix fused multiply-add (MFMA) intrinsic. On gfx94x,
+ the `blgp` immarg is a NEG bitfield rather than a B-lane permutation.
+
+ Example:
+ ```mlir
+ %r0 = }] # mnemonic # [{ %a0, %b0, %c0, 0, 0, neg_a|neg_b : (}]
+ # ABType.typeName # [{, }] # ABType.typeName # [{, }] # CDType.typeName
+ # [{) -> }] # CDType.typeName # [{
+ ```
+ }];
+}
+
class ROCDL_Mfma_Scale_IntrOp<string mnemonic, Type AB, Type CD> :
ROCDL_IntrOp<mnemonic, [], [0, 1], [], 1, 0, 0, 0, [3, 4, 5, 7], ["cbsz", "blgp", "opselA", "opselB"]>,
Arguments<(ins
LLVM_VectorOf<AB>:$a,
LLVM_VectorOf<AB>:$b,
LLVM_VectorOf<CD>:$c,
- I32Attr:$cbsz,
- I32Attr:$blgp,
+ ROCDL_MatrixFormatAttr:$cbsz,
+ ROCDL_MatrixFormatAttr:$blgp,
I32Attr:$opselA,
I32:$scaleA,
I32Attr:$opselB,
@@ -1003,15 +883,15 @@ class ROCDL_Mfma_Scale_IntrOp<string mnemonic, Type AB, Type CD> :
Example:
```mlir
// Scaled MFMA with fp8 * fp8 inputs.
- %r0 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %a, %c, 0, 0, 0, %scaleA, 0, %scaleB :
+ %r0 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %a, %c, fp8_e4m3, fp8_e4m3, 0, %scaleA, 0, %scaleB :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// Scaled MFMA with fp8 * bf8 inputs.
- %r1 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %a, %c, 0, 1, 0, %scaleA, 0, %scaleB :
+ %r1 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %a, %c, fp8_e4m3, fp8_e5m2, 0, %scaleA, 0, %scaleB :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// Scaled MFMA with fp8 * fp6 inputs (6xi32 operand B).
- %r2 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %b6, %c, 0, 2, 0, %scaleA, 0, %scaleB :
+ %r2 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %a, %b6, %c, fp8_e4m3, fp6_e2m3, 0, %scaleA, 0, %scaleB :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
```
}];
@@ -1085,8 +965,8 @@ def ROCDL_mfma_f32_32x32x8bf16_1k : ROCDL_Mfma_IntrOp<"mfma.f32.32x32x8bf16.1k",
def ROCDL_mfma_f32_16x16x16bf16_1k : ROCDL_Mfma_IntrOp<"mfma.f32.16x16x16bf16.1k", ROCDL_ConcreteVector<I16, 4>, ROCDL_ConcreteVector<F32, 4>>;
// Note: in gfx94x, unlike in gfx90a, the f64 xdlops use the "blgp" argument as
// a NEG bitfield. See IntrinsicsAMDGPU.td for more info.
-def ROCDL_mfma_f64_16x16x4f64 : ROCDL_Mfma_IntrOp<"mfma.f64.16x16x4f64", ROCDL_Scalar<F64>, ROCDL_ConcreteVector<F64, 4>>;
-def ROCDL_mfma_f64_4x4x4f64 : ROCDL_Mfma_IntrOp<"mfma.f64.4x4x4f64", ROCDL_Scalar<F64>, ROCDL_Scalar<F64>>;
+def ROCDL_mfma_f64_16x16x4f64 : ROCDL_Mfma_F64_IntrOp<"mfma.f64.16x16x4f64", ROCDL_Scalar<F64>, ROCDL_ConcreteVector<F64, 4>>;
+def ROCDL_mfma_f64_4x4x4f64 : ROCDL_Mfma_F64_IntrOp<"mfma.f64.4x4x4f64", ROCDL_Scalar<F64>, ROCDL_Scalar<F64>>;
// New in gfx94x.
def ROCDL_mfma_i32_16x16x32_i8 : ROCDL_Mfma_IntrOp<"mfma.i32.16x16x32.i8", ROCDL_Scalar<I64>, ROCDL_ConcreteVector<I32, 4>>;
def ROCDL_mfma_i32_32x32x16_i8 : ROCDL_Mfma_IntrOp<"mfma.i32.32x32x16.i8", ROCDL_Scalar<I64>, ROCDL_ConcreteVector<I32, 16>>;
@@ -1224,13 +1104,15 @@ class ROCDL_WMMA_ModsAll_Reuse_IntrOp<string mnemonic, Type AB, Type CD> : ROCDL
LLVM_ScalarOrVectorOf<AB>:$a,
DefaultValuedAttr<I1Attr, "0">:$signB,
LLVM_ScalarOrVectorOf<AB>:$b,
- DefaultValuedAttr<I16Attr, "0">:$modC,
+ DefaultValuedAttr<ROCDL_WMMACModifierAttr,
+ "::mlir::ROCDL::WMMACModifier::none">:$modC,
LLVM_ScalarOrVectorOf<CD>:$c,
DefaultValuedAttr<I1Attr, "0">:$reuseA,
DefaultValuedAttr<I1Attr, "0">:$reuseB)> {
let results = (outs LLVM_ScalarOrVectorOf<CD>:$res);
let assemblyFormat = [{
- $a `,` $b `,` $c attr-dict `:` functional-type(operands, $res)
+ $a `,` $b `,` $c `,` `modC` `=` $modC attr-dict `:`
+ functional-type(operands, $res)
}];
let description = [{
Wave Matrix Multiply-Accumulate (WMMA) with sign, modC, and reuse controls.
@@ -1238,7 +1120,7 @@ class ROCDL_WMMA_ModsAll_Reuse_IntrOp<string mnemonic, Type AB, Type CD> : ROCDL
Example:
```mlir
// WMMA f32 with f16 inputs and reuse controls.
- %r = rocdl.wmma.f32.16x16x32.f16 %a, %b, %c :
+ %r = rocdl.wmma.f32.16x16x32.f16 %a, %b, %c, modC = none :
(vector<16xf16>, vector<16xf16>, vector<8xf32>) -> vector<8xf32>
```
}];
@@ -1249,13 +1131,15 @@ class ROCDL_WMMA_ModsC_IntrOp<string mnemonic, Type AB, Type CD> : ROCDL_IntrOp<
Arguments<(ins
LLVM_ScalarOrVectorOf<AB>:$a,
LLVM_ScalarOrVectorOf<AB>:$b,
- DefaultValuedAttr<I16Attr, "0">:$modC,
+ DefaultValuedAttr<ROCDL_WMMACModifierAttr,
+ "::mlir::ROCDL::WMMACModifier::none">:$modC,
LLVM_ScalarOrVectorOf<CD>:$c,
DefaultValuedAttr<I1Attr, "0">:$reuseA,
DefaultValuedAttr<I1Attr, "0">:$reuseB)> {
let results = (outs LLVM_ScalarOrVectorOf<CD>:$res);
let assemblyFormat = [{
- $a `,` $b `,` $c attr-dict `:` functional-type(operands, $res)
+ $a `,` $b `,` $c `,` `modC` `=` $modC attr-dict `:`
+ functional-type(operands, $res)
}];
let description = [{
Wave Matrix Multiply-Accumulate (WMMA) with modC and reuse controls.
@@ -1263,7 +1147,7 @@ class ROCDL_WMMA_ModsC_IntrOp<string mnemonic, Type AB, Type CD> : ROCDL_IntrOp<
Example:
```mlir
// WMMA f32 with fp8 inputs and modC/reuse controls.
- %r = rocdl.wmma.f32.16x16x64.fp8_fp8 %a, %b, %c :
+ %r = rocdl.wmma.f32.16x16x64.fp8_fp8 %a, %b, %c, modC = none :
(vector<16xi32>, vector<16xi32>, vector<8xf32>) -> vector<8xf32>
```
}];
@@ -1274,13 +1158,15 @@ class ROCDL_WMMA_ModsC_Diff_IntrOp<string mnemonic, Type AB, Type C, Type D> : R
Arguments<(ins
LLVM_ScalarOrVectorOf<AB>:$a,
LLVM_ScalarOrVectorOf<AB>:$b,
- DefaultValuedAttr<I16Attr, "0">:$modC,
+ DefaultValuedAttr<ROCDL_WMMACModifierAttr,
+ "::mlir::ROCDL::WMMACModifier::none">:$modC,
LLVM_ScalarOrVectorOf<C>:$c,
DefaultValuedAttr<I1Attr, "0">:$reuseA,
DefaultValuedAttr<I1Attr, "0">:$reuseB)> {
let results = (outs LLVM_ScalarOrVectorOf<D>:$res);
let assemblyFormat = [{
- $a `,` $b `,` $c attr-dict `:` functional-type(operands, $res)
+ $a `,` $b `,` $c `,` `modC` `=` $modC attr-dict `:`
+ functional-type(operands, $res)
}];
let description = [{
Wave Matrix Multiply-Accumulate (WMMA) with
diff erent C and D types.
@@ -1288,7 +1174,7 @@ class ROCDL_WMMA_ModsC_Diff_IntrOp<string mnemonic, Type AB, Type C, Type D> : R
Example:
```mlir
// WMMA bf16 output from f32 accumulator with bf16 inputs.
- %r = rocdl.wmma.bf16f32.16x16x32.bf16 %a, %b, %c :
+ %r = rocdl.wmma.bf16f32.16x16x32.bf16 %a, %b, %c, modC = none :
(vector<16xbf16>, vector<16xbf16>, vector<8xf32>) -> vector<16xbf16>
```
}];
@@ -1330,23 +1216,35 @@ class ROCDL_WMMA_Scale_IntrOp<string mnemonic, Type AB, Type CD, Type ScaleExpTy
["fmtA", "fmtB", "modC", "scaleAType", "fmtScaleA",
"scaleBType", "fmtScaleB", "reuseA", "reuseB"]>,
Arguments<(ins
- DefaultValuedAttr<I32Attr, "0">:$fmtA,
+ DefaultValuedAttr<ROCDL_MatrixFormatAttr,
+ "::mlir::ROCDL::MatrixFormat::fp8_e4m3">:$fmtA,
LLVM_ScalarOrVectorOf<AB>:$a,
- DefaultValuedAttr<I32Attr, "0">:$fmtB,
+ DefaultValuedAttr<ROCDL_MatrixFormatAttr,
+ "::mlir::ROCDL::MatrixFormat::fp8_e4m3">:$fmtB,
LLVM_ScalarOrVectorOf<AB>:$b,
- DefaultValuedAttr<I16Attr, "0">:$modC,
+ DefaultValuedAttr<ROCDL_WMMACModifierAttr,
+ "::mlir::ROCDL::WMMACModifier::none">:$modC,
LLVM_ScalarOrVectorOf<CD>:$c,
- DefaultValuedAttr<I32Attr, "0">:$scaleAType,
- DefaultValuedAttr<I32Attr, "0">:$fmtScaleA,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleAttr,
+ "::mlir::ROCDL::WMMAMatrixScale::row0">:$scaleAType,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleFormatAttr,
+ "::mlir::ROCDL::WMMAMatrixScaleFormat::e8">:$fmtScaleA,
ScaleExpTy:$scaleA,
- DefaultValuedAttr<I32Attr, "0">:$scaleBType,
- DefaultValuedAttr<I32Attr, "0">:$fmtScaleB,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleAttr,
+ "::mlir::ROCDL::WMMAMatrixScale::row0">:$scaleBType,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleFormatAttr,
+ "::mlir::ROCDL::WMMAMatrixScaleFormat::e8">:$fmtScaleB,
ScaleExpTy:$scaleB,
DefaultValuedAttr<I1Attr, "0">:$reuseA,
DefaultValuedAttr<I1Attr, "0">:$reuseB)> {
let results = (outs LLVM_ScalarOrVectorOf<CD>:$res);
let assemblyFormat = [{
- $a `,` $b `,` $c `,` $scaleA `,` $scaleB attr-dict `:` functional-type(operands, $res)
+ $a `,` $b `,` $c `,` $scaleA `,` $scaleB `,`
+ `fmtA` `=` $fmtA `,` `fmtB` `=` $fmtB `,`
+ `modC` `=` $modC `,`
+ `scaleAType` `=` $scaleAType `,` `fmtScaleA` `=` $fmtScaleA `,`
+ `scaleBType` `=` $scaleBType `,` `fmtScaleB` `=` $fmtScaleB
+ attr-dict `:` functional-type(operands, $res)
}];
let description = [{
Scaled Wave Matrix Multiply-Accumulate (WMMA) with per-operand scaling.
@@ -1354,7 +1252,9 @@ class ROCDL_WMMA_Scale_IntrOp<string mnemonic, Type AB, Type CD, Type ScaleExpTy
Example:
```mlir
// Scaled WMMA with f8f6f4 format inputs.
- %r = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %a, %b, %c, %scaleA, %scaleB :
+ %r = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %a, %b, %c, %scaleA, %scaleB
+ fmtA = fp8_e4m3, fmtB = fp8_e4m3, modC = none,
+ scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 :
(vector<16xi32>, vector<16xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
```
}];
@@ -1367,19 +1267,28 @@ class ROCDL_WMMA_Scale_F4_IntrOp<string mnemonic, Type AB, Type CD, Type ScaleEx
Arguments<(ins
LLVM_ScalarOrVectorOf<AB>:$a,
LLVM_ScalarOrVectorOf<AB>:$b,
- DefaultValuedAttr<I16Attr, "0">:$modC,
+ DefaultValuedAttr<ROCDL_WMMACModifierAttr,
+ "::mlir::ROCDL::WMMACModifier::none">:$modC,
LLVM_ScalarOrVectorOf<CD>:$c,
- DefaultValuedAttr<I32Attr, "0">:$scaleAType,
- DefaultValuedAttr<I32Attr, "0">:$fmtScaleA,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleAttr,
+ "::mlir::ROCDL::WMMAMatrixScale::row0">:$scaleAType,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleFormatAttr,
+ "::mlir::ROCDL::WMMAMatrixScaleFormat::e8">:$fmtScaleA,
ScaleExpTy:$scaleA,
- DefaultValuedAttr<I32Attr, "0">:$scaleBType,
- DefaultValuedAttr<I32Attr, "0">:$fmtScaleB,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleAttr,
+ "::mlir::ROCDL::WMMAMatrixScale::row0">:$scaleBType,
+ DefaultValuedAttr<ROCDL_WMMAMatrixScaleFormatAttr,
+ "::mlir::ROCDL::WMMAMatrixScaleFormat::e8">:$fmtScaleB,
ScaleExpTy:$scaleB,
DefaultValuedAttr<I1Attr, "0">:$reuseA,
DefaultValuedAttr<I1Attr, "0">:$reuseB)> {
let results = (outs LLVM_ScalarOrVectorOf<CD>:$res);
let assemblyFormat = [{
- $a `,` $b `,` $c `,` $scaleA `,` $scaleB attr-dict `:` functional-type(operands, $res)
+ $a `,` $b `,` $c `,` $scaleA `,` $scaleB `,`
+ `modC` `=` $modC `,`
+ `scaleAType` `=` $scaleAType `,` `fmtScaleA` `=` $fmtScaleA `,`
+ `scaleBType` `=` $scaleBType `,` `fmtScaleB` `=` $fmtScaleB
+ attr-dict `:` functional-type(operands, $res)
}];
let description = [{
Scaled Wave Matrix Multiply-Accumulate (WMMA) for F4 format inputs.
@@ -1387,7 +1296,9 @@ class ROCDL_WMMA_Scale_F4_IntrOp<string mnemonic, Type AB, Type CD, Type ScaleEx
Example:
```mlir
// Scaled WMMA with f4 format inputs.
- %r = rocdl.wmma.scale.f32.16x16x128.f4 %a, %b, %c, %scaleA, %scaleB :
+ %r = rocdl.wmma.scale.f32.16x16x128.f4 %a, %b, %c, %scaleA, %scaleB
+ modC = none, scaleAType = row0, fmtScaleA = e8,
+ scaleBType = row0, fmtScaleB = e8 :
(vector<8xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
```
}];
@@ -1507,7 +1418,7 @@ class ROCDL_Sudot_IntrOp<string mnemonic> :
the same for `b`. `clamp` controls result clamping.
These ops correspond to RDNA's unified mixed-sign `v_dot4_i32_iu8`
- and `v_dot8_i32_iu4` instructions (gfx11+).
+ and `v_dot8_i32_iu4` instructions (gfx11+).
Example:
```mlir
@@ -1779,10 +1690,11 @@ def ROCDL_LoadToLDSOp :
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$size,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_PreGfx12CachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $size `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $size `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict `:` type($globalPtr)
}];
let extraClassDefinition = [{
@@ -1798,10 +1710,11 @@ def ROCDL_LoadAsyncToLDSOp :
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$size,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_PreGfx12CachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $size `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $size `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict `:` qualified(type($globalPtr)) `,` qualified(type($ldsPtr))
}];
let extraClassDefinition = [{
@@ -1844,10 +1757,11 @@ def ROCDL_GlobalLoadLDSOp :
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$size,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_PreGfx12CachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $size `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $size `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict
}];
let extraClassDefinition = [{
@@ -1863,10 +1777,11 @@ def ROCDL_GlobalLoadAsyncLDSOp :
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$size,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_PreGfx12CachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $size `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $size `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict `:` qualified(type($globalPtr)) `,` qualified(type($ldsPtr))
}];
let extraClassDefinition = [{
@@ -1904,10 +1819,11 @@ foreach bitsVal = [8, 32, 64, 128] in {
dag args = (ins Arg<ROCDLGlobalBuffer, "", [MemRead]>:$globalPtr,
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict `:` qualified(type($globalPtr)) `,` qualified(type($ldsPtr))
}];
let description = [{
@@ -1942,10 +1858,11 @@ foreach bitsVal = [8, 32, 64, 128] in {
dag args = (ins Arg<ROCDLGlobalBuffer, "", [MemWrite]>:$globalPtr,
Arg<ROCDLBufferLDS, "", [MemRead]>:$ldsPtr,
I32Attr:$offset,
- I32Attr:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $offset `,` $aux
+ $globalPtr `,` $ldsPtr `,` $offset `,`
+ custom<CachePolicy>($aux)
attr-dict `:` qualified(type($globalPtr)) `,` qualified(type($ldsPtr))
}];
let description = [{
@@ -1976,11 +1893,12 @@ foreach bitsVal = [8, 32, 64, 128] in {
dag args = (ins Arg<ROCDLGlobalBuffer, "", [MemRead]>:$globalPtr,
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32Attr:$offset,
- I32Attr:$cpol,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$cpol,
I32:$mask);
let arguments = !con(args, baseArgs);
let assemblyFormat = [{
- $globalPtr `,` $ldsPtr `,` $offset `,` $cpol `,` $mask
+ $globalPtr `,` $ldsPtr `,` $offset `,`
+ custom<CachePolicy>($cpol) `,` $mask
attr-dict `:` qualified(type($globalPtr)) `,` qualified(type($ldsPtr))
}];
let description = [{
@@ -2012,7 +1930,8 @@ class ROCDL_TensorLDSIntrOp<string mnemonic> :
ROCDL_IntrOp<mnemonic, [], [], [], 0, 0, 1, 0, [5], ["cachePolicy"]> {
dag args = (ins ROCDL_V4I32Type:$dgroup0, ROCDL_V8I32Type:$dgroup1,
ROCDL_V4I32Type:$dgroup2, ROCDL_V4I32Type:$dgroup3,
- ROCDL_V8I32Type:$dgroup4, I32Attr:$cachePolicy);
+ ROCDL_V8I32Type:$dgroup4,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$cachePolicy);
let arguments = !con(args, baseArgs);
let summary = "Base class for ROCDL tensor load/store to/from LDS.";
let description = [{
@@ -2026,14 +1945,16 @@ class ROCDL_TensorLDSIntrOp<string mnemonic> :
Example:
```mlir
// Tensor load from global memory to LDS using 4 descriptor groups.
- rocdl.tensor.load.to.lds %dg0, %dg1, %dg2, %dg3 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.load.to.lds %dg0, %dg1, %dg2, %dg3, %dg4, 0 : vector<4xi32>, vector<8xi32>
// Tensor store from LDS to global memory using 4 descriptor groups.
- rocdl.tensor.store.from.lds %dg0, %dg1, %dg2, %dg3 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.store.from.lds %dg0, %dg1, %dg2, %dg3, %dg4, 0 : vector<4xi32>, vector<8xi32>
```
}];
let assemblyFormat = [{
- attr-dict operands `cachepolicy` $cachePolicy `:` type($dgroup0) `,` type($dgroup1)
+ $dgroup0 `,` $dgroup1 `,` $dgroup2 `,` $dgroup3 `,` $dgroup4 `,`
+ custom<CachePolicy>($cachePolicy) attr-dict
+ `:` type($dgroup0) `,` type($dgroup1)
}];
let extraClassDefinition = [{
SmallVector<Value> $cppClass::getAccessedOperands() {
@@ -2064,13 +1985,13 @@ def ROCDL_MakeBufferRsrcOp :
}
def ROCDL_RawPtrBufferLoadOp :
- ROCDL_IntrOp<"raw.ptr.buffer.load", [0], [], [], 1, 0, 1> {
+ ROCDL_IntrOp<"raw.ptr.buffer.load", [0], [], [], 1, 0, 1, 0, [3], ["aux"]> {
dag args = (ins Arg<ROCDLBufferRsrc, "", [MemRead]>:$rsrc,
I32:$offset,
I32:$soffset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict `:` type($res)";
+ let assemblyFormat = "$rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($res)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2079,16 +2000,16 @@ def ROCDL_RawPtrBufferLoadOp :
}
def ROCDL_RawPtrBufferLoadLdsOp :
- ROCDL_IntrOp<"raw.ptr.buffer.load.lds", [], [], [], 0, 0, 1> {
+ ROCDL_IntrOp<"raw.ptr.buffer.load.lds", [], [], [], 0, 0, 1, 0, [6], ["aux"]> {
dag args = (ins Arg<ROCDLBufferRsrc, "", [MemRead]>:$rsrc,
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32:$size,
I32:$voffset,
I32:$soffset,
I32:$offset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict";
+ let assemblyFormat = "$rsrc `,` $ldsPtr `,` $size `,` $voffset `,` $soffset `,` $offset `,` custom<CachePolicy>($aux) attr-dict";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc(), getLdsPtr()};
@@ -2097,16 +2018,16 @@ def ROCDL_RawPtrBufferLoadLdsOp :
}
def ROCDL_RawPtrBufferLoadAsyncLdsOp :
- ROCDL_IntrOp<"raw.ptr.buffer.load.async.lds", [], [], [], 0, 0, 1> {
+ ROCDL_IntrOp<"raw.ptr.buffer.load.async.lds", [], [], [], 0, 0, 1, 0, [6], ["aux"]> {
dag args = (ins Arg<ROCDLBufferRsrc, "", [MemRead]>:$rsrc,
Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
I32:$size,
I32:$voffset,
I32:$soffset,
I32:$offset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_PreGfx12CachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict";
+ let assemblyFormat = "$rsrc `,` $ldsPtr `,` $size `,` $voffset `,` $soffset `,` $offset `,` custom<CachePolicy>($aux) attr-dict";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc(), getLdsPtr()};
@@ -2125,20 +2046,20 @@ def ROCDL_RawPtrBufferLoadAsyncLdsOp :
Example:
```mlir
// Async buffer load to LDS via buffer resource pointer.
- rocdl.raw.ptr.buffer.load.async.lds %rsrc, %ldsPtr, %size, %voffset, %soffset, %offset, %aux
+ rocdl.raw.ptr.buffer.load.async.lds %rsrc, %ldsPtr, %size, %voffset, %soffset, %offset, 0
```
}];
}
def ROCDL_RawPtrBufferStoreOp :
- ROCDL_IntrOp<"raw.ptr.buffer.store", [], [0], [], 0, 0, 1> {
+ ROCDL_IntrOp<"raw.ptr.buffer.store", [], [0], [], 0, 0, 1, 0, [4], ["aux"]> {
dag args = (ins LLVM_Type:$vdata,
Arg<ROCDLBufferRsrc, "", [MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict `:` type($vdata)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2149,15 +2070,15 @@ def ROCDL_RawPtrBufferStoreOp :
def ROCDL_RawPtrBufferAtomicCmpSwap :
ROCDL_IntrOp<"raw.ptr.buffer.atomic.cmpswap",
- [0], [], [AllTypesMatch<["res", "src", "cmp"]>], 1, 0, 1> {
+ [0], [], [AllTypesMatch<["res", "src", "cmp"]>], 1, 0, 1, 0, [5], ["aux"]> {
dag args = (ins LLVM_Type:$src,
LLVM_Type:$cmp,
Arg<ROCDLBufferRsrc, "", [MemRead, MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict `:` type($res)";
+ let assemblyFormat = "$src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($res)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2167,14 +2088,14 @@ def ROCDL_RawPtrBufferAtomicCmpSwap :
class ROCDL_RawPtrBufferAtomicRet<string op> :
ROCDL_IntrOp<"raw.ptr.buffer.atomic." # op, [0], [],
- [AllTypesMatch<["res", "vdata"]>], 1, 0, 1> {
+ [AllTypesMatch<["res", "vdata"]>], 1, 0, 1, 0, [4], ["aux"]> {
dag args = (ins LLVM_Type:$vdata,
Arg<ROCDLBufferRsrc, "", [MemRead, MemWrite]>:$rsrc,
I32:$offset,
I32:$soffset,
- I32:$aux);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux);
let arguments = !con(args, baseArgs);
- let assemblyFormat = "operands attr-dict `:` type($res)";
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
let extraClassDefinition = [{
::llvm::SmallVector<::mlir::Value> $cppClass::getAccessedOperands() {
return {getRsrc()};
@@ -2192,34 +2113,36 @@ def ROCDL_RawPtrBufferAtomicFaddOp : ROCDL_RawPtrBufferAtomicRet<"fadd">;
// Raw buffer load/store intrinsics
def ROCDL_RawBufferLoadOp :
- ROCDL_IntrOp<"raw.buffer.load", [0], [], [], 1>,
- Arguments<(ins LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)> {
- let hasCustomAssemblyFormat = 1;
+ ROCDL_IntrOp<"raw.buffer.load", [0], [], [], 1, 0, 0, 0, [3], ["aux"]>,
+ Arguments<(ins ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux)> {
+ let assemblyFormat = "$rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($res)";
}
def ROCDL_RawBufferStoreOp :
- ROCDL_IntrOp<"raw.buffer.store", [], [0], [], 0>,
+ ROCDL_IntrOp<"raw.buffer.store", [], [0], [], 0, 0, 0, 0, [4], ["aux"]>,
Arguments<(ins LLVM_Type:$vdata,
- LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)>{
- let hasCustomAssemblyFormat = 1;
+ ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_NonAtomicBufferCachePolicyCompatAttr>:$aux)>{
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
}
def ROCDL_RawBufferAtomicCmpSwap :
- ROCDL_IntrOp<"raw.buffer.atomic.cmpswap", [], [0], [AllTypesMatch<["res", "src", "cmp"]>], 1>,
+ ROCDL_IntrOp<"raw.buffer.atomic.cmpswap", [], [0], [AllTypesMatch<["res", "src", "cmp"]>],
+ 1, 0, 0, 0, [5], ["aux"]>,
Arguments<(ins LLVM_Type:$src,
LLVM_Type:$cmp,
- LLVM_Type:$rsrc,
+ ROCDL_V4I32Type:$rsrc,
I32:$offset,
I32:$soffset,
- I32:$aux)>{
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux)>{
let assemblyFormat = [{
- attr-dict `(` operands `)` `:` type($res) `,` type($rsrc)
+ attr-dict `(` $src `,` $cmp `,` $rsrc `,` $offset `,` $soffset `,`
+ custom<CachePolicy>($aux) `)` `:` type($res) `,` type($rsrc)
}];
}
@@ -2227,9 +2150,9 @@ def ROCDL_RawBufferAtomicCmpSwap :
// Memory prefetch intrinsics
def ROCDL_GlobalPrefetchOp :
- ROCDL_IntrOp<"global.prefetch", [], [], [], 0, 0, 1, 0, [1], ["scope"]> {
+ ROCDL_IntrOp<"global.prefetch", [], [], [], 0, 0, 1, 0, [1], ["cachePolicy"]> {
dag args = (ins Arg<LLVM_PointerInAddressSpace<1>, "", [MemWrite, MemRead]>:$ptr,
- I32Attr:$scope);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$cachePolicy);
let arguments = !con(args, baseArgs);
let description = [{
Prefetches 1 byte of data per lane from global memory into the WGP-cache or L2-cache.
@@ -2238,11 +2161,11 @@ def ROCDL_GlobalPrefetchOp :
Example:
```mlir
// Prefetch from global memory into cache.
- rocdl.global.prefetch %ptr, scope 0 : !llvm.ptr<1>
+ rocdl.global.prefetch %ptr, 0 : !llvm.ptr<1>
```
}];
let results = (outs);
- let assemblyFormat = "$ptr `,` `scope` $scope attr-dict `:` qualified(type($ptr))";
+ let assemblyFormat = "$ptr `,` custom<CachePolicy>($cachePolicy) attr-dict `:` qualified(type($ptr))";
let extraClassDefinition = [{
SmallVector<Value> $cppClass::getAccessedOperands() {
return {getPtr()};
@@ -2251,9 +2174,9 @@ def ROCDL_GlobalPrefetchOp :
}
def ROCDL_FlatPrefetchOp :
- ROCDL_IntrOp<"flat.prefetch", [], [], [], 0, 0, 1, 0, [1], ["scope"]> {
+ ROCDL_IntrOp<"flat.prefetch", [], [], [], 0, 0, 1, 0, [1], ["cachePolicy"]> {
dag args = (ins Arg<LLVM_PointerInAddressSpace<0>, "", [MemWrite, MemRead]>:$ptr,
- I32Attr:$scope);
+ ROCDL_DefaultCachePolicyAttr<ROCDL_Gfx12NonAtomicCachePolicyCompatAttr>:$cachePolicy);
let arguments = !con(args, baseArgs);
let description = [{
Prefetches 1 byte of data per lane using flat-memory addresses into the WGP-cache or L2-cache.
@@ -2262,11 +2185,11 @@ def ROCDL_FlatPrefetchOp :
Example:
```mlir
// Prefetch from flat memory into cache.
- rocdl.flat.prefetch %ptr, scope 0 : !llvm.ptr
+ rocdl.flat.prefetch %ptr, 0 : !llvm.ptr
```
}];
let results = (outs);
- let assemblyFormat = "$ptr `,` `scope` $scope attr-dict `:` qualified(type($ptr))";
+ let assemblyFormat = "$ptr `,` custom<CachePolicy>($cachePolicy) attr-dict `:` qualified(type($ptr))";
let extraClassDefinition = [{
SmallVector<Value> $cppClass::getAccessedOperands() {
return {getPtr()};
@@ -2333,13 +2256,13 @@ def ROCDL_DsAtomicAsyncBarrierArriveOp :
def ROCDL_RawBufferAtomicFAddOp :
ROCDL_IntrOp<"raw.buffer.atomic.fadd", [0], [],
- [AllTypesMatch<["res", "vdata"]>], 1>,
+ [AllTypesMatch<["res", "vdata"]>], 1, 0, 0, 0, [4], ["aux"]>,
Arguments<(ins LLVM_Type:$vdata,
- LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)>{
- let hasCustomAssemblyFormat = 1;
+ ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux)>{
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
}
//===---------------------------------------------------------------------===//
@@ -2347,13 +2270,13 @@ def ROCDL_RawBufferAtomicFAddOp :
def ROCDL_RawBufferAtomicFMaxOp :
ROCDL_IntrOp<"raw.buffer.atomic.fmax", [0], [],
- [AllTypesMatch<["res", "vdata"]>], 1>,
+ [AllTypesMatch<["res", "vdata"]>], 1, 0, 0, 0, [4], ["aux"]>,
Arguments<(ins LLVM_Type:$vdata,
- LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)>{
- let hasCustomAssemblyFormat = 1;
+ ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux)>{
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
}
//===---------------------------------------------------------------------===//
@@ -2361,13 +2284,13 @@ def ROCDL_RawBufferAtomicFMaxOp :
def ROCDL_RawBufferAtomicSMaxOp :
ROCDL_IntrOp<"raw.buffer.atomic.smax", [0], [],
- [AllTypesMatch<["res", "vdata"]>], 1>,
+ [AllTypesMatch<["res", "vdata"]>], 1, 0, 0, 0, [4], ["aux"]>,
Arguments<(ins LLVM_Type:$vdata,
- LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)>{
- let hasCustomAssemblyFormat = 1;
+ ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux)>{
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
}
//===---------------------------------------------------------------------===//
@@ -2375,13 +2298,13 @@ def ROCDL_RawBufferAtomicSMaxOp :
def ROCDL_RawBufferAtomicUMinOp :
ROCDL_IntrOp<"raw.buffer.atomic.umin", [0], [],
- [AllTypesMatch<["res", "vdata"]>], 1>,
+ [AllTypesMatch<["res", "vdata"]>], 1, 0, 0, 0, [4], ["aux"]>,
Arguments<(ins LLVM_Type:$vdata,
- LLVM_Type:$rsrc,
- LLVM_Type:$offset,
- LLVM_Type:$soffset,
- LLVM_Type:$aux)>{
- let hasCustomAssemblyFormat = 1;
+ ROCDL_V4I32Type:$rsrc,
+ I32:$offset,
+ I32:$soffset,
+ ROCDL_DefaultCachePolicyAttr<ROCDL_AtomicBufferCachePolicyCompatAttr>:$aux)>{
+ let assemblyFormat = "$vdata `,` $rsrc `,` $offset `,` $soffset `,` custom<CachePolicy>($aux) attr-dict `:` type($vdata)";
}
// DPP Update intrinsic
diff --git a/mlir/include/mlir/Target/LLVM/ROCDL/Utils.h b/mlir/include/mlir/Target/LLVM/ROCDL/Utils.h
index 7f64f280cd507..f189fc26e5a75 100644
--- a/mlir/include/mlir/Target/LLVM/ROCDL/Utils.h
+++ b/mlir/include/mlir/Target/LLVM/ROCDL/Utils.h
@@ -19,6 +19,7 @@
#include "mlir/IR/Attributes.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Target/LLVM/ModuleToObject.h"
+#include "llvm/ADT/BitmaskEnum.h"
namespace mlir {
namespace ROCDL {
@@ -43,6 +44,8 @@ enum class AMDGCNLibraries : uint32_t {
All = (LastLib << 1) - 1
};
+LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
+
/// Assembles ISA to an object code.
FailureOr<SmallVector<char, 0>>
assembleIsa(StringRef isa, StringRef targetTriple, StringRef chip,
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index f7dbb81000386..90e15b1680446 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -488,16 +488,12 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
sgprOffset = LLVM::MulOp::create(rewriter, loc, sgprOffset, byteWidthConst);
args.push_back(sgprOffset);
- // bit 0: GLC = 0 (atomics drop value, less coherency)
- // bits 1-2: SLC, DLC = 0 (similarly)
- // bit 3: swizzled (0 for raw)
- // Note: atomic ret/no-ret bit set by backend based on actual usage.
- args.push_back(createI32Constant(rewriter, loc, 0));
-
llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
llvmBufferValType);
- Operation *lowered = Intrinsic::create(rewriter, loc, resultTypes, args,
- ArrayRef<NamedAttribute>());
+ typename Intrinsic::Properties properties;
+ properties.aux = rewriter.getI32IntegerAttr(0);
+ Operation *lowered =
+ Intrinsic::create(rewriter, loc, resultTypes, args, properties);
if (lowered->getNumResults() == 1) {
Value replacement = lowered->getResult(0);
if (llvmBufferValType != llvmWantedDataType) {
@@ -689,8 +685,7 @@ struct SchedBarrierOpLowering : public ConvertOpToLLVMPattern<SchedBarrierOp> {
LogicalResult
matchAndRewrite(SchedBarrierOp op, SchedBarrierOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- rewriter.replaceOpWithNewOp<ROCDL::SchedBarrier>(op,
- (uint32_t)op.getOpts());
+ rewriter.replaceOpWithNewOp<ROCDL::SchedBarrier>(op, op.getOptsAttr());
return success();
}
};
@@ -792,10 +787,11 @@ static Value castScaleOperand(ConversionPatternRewriter &rewriter, Location loc,
}
/// Maps f8 scale element types to WMMA scale format codes.
-static std::optional<uint32_t> getWmmaScaleFormat(Type elemType) {
- return TypeSwitch<Type, std::optional<uint32_t>>(elemType)
- .Case([](Float8E8M0FNUType) { return 0; })
- .Case([](Float8E4M3FNType) { return 2; })
+static std::optional<ROCDL::WMMAMatrixScaleFormat>
+getWmmaScaleFormat(Type elemType) {
+ return TypeSwitch<Type, std::optional<ROCDL::WMMAMatrixScaleFormat>>(elemType)
+ .Case([](Float8E8M0FNUType) { return ROCDL::WMMAMatrixScaleFormat::e8; })
+ .Case([](Float8E4M3FNType) { return ROCDL::WMMAMatrixScaleFormat::e4m3; })
.Default(std::nullopt);
}
@@ -1061,13 +1057,15 @@ static std::optional<StringRef> mfmaOpToIntrinsic(MFMAOp mfma,
return std::nullopt;
}
-static std::optional<uint32_t> smallFloatTypeToFormatCode(Type mlirElemType) {
- return llvm::TypeSwitch<Type, std::optional<uint32_t>>(mlirElemType)
- .Case([](Float8E4M3FNType) { return 0u; })
- .Case([](Float8E5M2Type) { return 1u; })
- .Case([](Float6E2M3FNType) { return 2u; })
- .Case([](Float6E3M2FNType) { return 3u; })
- .Case([](Float4E2M1FNType) { return 4u; })
+static std::optional<ROCDL::MatrixFormat>
+smallFloatTypeToMatrixFormat(Type mlirElemType) {
+ return llvm::TypeSwitch<Type, std::optional<ROCDL::MatrixFormat>>(
+ mlirElemType)
+ .Case([](Float8E4M3FNType) { return ROCDL::MatrixFormat::fp8_e4m3; })
+ .Case([](Float8E5M2Type) { return ROCDL::MatrixFormat::fp8_e5m2; })
+ .Case([](Float6E2M3FNType) { return ROCDL::MatrixFormat::fp6_e2m3; })
+ .Case([](Float6E3M2FNType) { return ROCDL::MatrixFormat::fp6_e3m2; })
+ .Case([](Float4E2M1FNType) { return ROCDL::MatrixFormat::fp4_e2m1; })
.Default(std::nullopt);
}
@@ -1078,7 +1076,10 @@ static std::optional<uint32_t> smallFloatTypeToFormatCode(Type mlirElemType) {
/// that intrinsic. Note that this is also used to implement some un-scaled
/// MFMAs, since the compiler represents the ordinary instruction as a "scaled"
/// MFMA with a scale of 0.
-static std::optional<std::tuple<StringRef, uint32_t, uint32_t>>
+using ScaledMFMAIntrinsic =
+ std::tuple<StringRef, ROCDL::MatrixFormat, ROCDL::MatrixFormat>;
+
+static std::optional<ScaledMFMAIntrinsic>
mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
uint32_t n, uint32_t k, uint32_t b, Chipset chipset) {
aType = getElementTypeOrSelf(aType);
@@ -1090,8 +1091,10 @@ mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
if (!isa<Float32Type>(destType))
return std::nullopt;
- std::optional<uint32_t> aTypeCode = smallFloatTypeToFormatCode(aType);
- std::optional<uint32_t> bTypeCode = smallFloatTypeToFormatCode(bType);
+ std::optional<ROCDL::MatrixFormat> aTypeCode =
+ smallFloatTypeToMatrixFormat(aType);
+ std::optional<ROCDL::MatrixFormat> bTypeCode =
+ smallFloatTypeToMatrixFormat(bType);
if (!aTypeCode || !bTypeCode)
return std::nullopt;
@@ -1106,7 +1109,7 @@ mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
return std::nullopt;
}
-static std::optional<std::tuple<StringRef, uint32_t, uint32_t>>
+static std::optional<ScaledMFMAIntrinsic>
mfmaOpToScaledIntrinsic(MFMAOp mfma, Chipset chipset) {
return mfmaOpToScaledIntrinsic(
mfma.getSourceA().getType(), mfma.getSourceB().getType(),
@@ -1114,7 +1117,7 @@ mfmaOpToScaledIntrinsic(MFMAOp mfma, Chipset chipset) {
mfma.getBlocks(), chipset);
}
-static std::optional<std::tuple<StringRef, uint32_t, uint32_t>>
+static std::optional<ScaledMFMAIntrinsic>
mfmaOpToScaledIntrinsic(ScaledMFMAOp smfma, Chipset chipset) {
return mfmaOpToScaledIntrinsic(smfma.getSourceA().getType(),
smfma.getSourceB().getType(),
@@ -1565,6 +1568,7 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
matchAndRewrite(MFMAOp op, MFMAOpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
Location loc = op.getLoc();
+ Type destElem = getElementTypeOrSelf(op.getDestD().getType());
Type outType = typeConverter->convertType(op.getDestD().getType());
Type intrinsicOutType = outType;
if (auto outVecType = dyn_cast<VectorType>(outType))
@@ -1581,8 +1585,8 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
op.getNegateA() | (op.getNegateB() << 1) | (op.getNegateC() << 2);
}
std::optional<StringRef> maybeIntrinsic = mfmaOpToIntrinsic(op, chipset);
- std::optional<std::tuple<StringRef, uint32_t, uint32_t>>
- maybeScaledIntrinsic = mfmaOpToScaledIntrinsic(op, chipset);
+ std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
+ mfmaOpToScaledIntrinsic(op, chipset);
if (!maybeIntrinsic.has_value() && !maybeScaledIntrinsic.has_value())
return op.emitOpError("no intrinsic matching MFMA size on given chipset");
@@ -1618,15 +1622,26 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
Value zero = createI32Constant(rewriter, loc, 0);
auto [_scaledName, aTypeCode, bTypeCode] = *maybeScaledIntrinsic;
loweredOp.addOperands({/*scale A=*/zero, /*scale B=*/zero});
- loweredOp.addAttributes({{"cbsz", rewriter.getI32IntegerAttr(aTypeCode)},
- {"blgp", rewriter.getI32IntegerAttr(bTypeCode)},
- {"opselA", rewriter.getI32IntegerAttr(0)},
- {"opselB", rewriter.getI32IntegerAttr(0)}});
+ loweredOp.addAttributes(
+ {{"cbsz",
+ ROCDL::MatrixFormatAttr::get(rewriter.getContext(), aTypeCode)},
+ {"blgp",
+ ROCDL::MatrixFormatAttr::get(rewriter.getContext(), bTypeCode)},
+ {"opselA", rewriter.getI32IntegerAttr(0)},
+ {"opselB", rewriter.getI32IntegerAttr(0)}});
} else {
+ Attribute blgpAttr =
+ destElem.isF64()
+ ? Attribute(ROCDL::MFMANegModifierAttr::get(
+ rewriter.getContext(),
+ static_cast<ROCDL::MFMANegModifier>(getBlgpField)))
+ : Attribute(ROCDL::MFMAPermBAttr::get(
+ rewriter.getContext(),
+ static_cast<ROCDL::MFMAPermB>(getBlgpField)));
loweredOp.addAttributes(
{{"cbsz", rewriter.getI32IntegerAttr(op.getCbsz())},
{"abid", rewriter.getI32IntegerAttr(op.getAbid())},
- {"blgp", rewriter.getI32IntegerAttr(getBlgpField)}});
+ {"blgp", blgpAttr}});
};
Value lowered = rewriter.create(loweredOp)->getResult(0);
if (outType != intrinsicOutType)
@@ -1650,8 +1665,8 @@ struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
if (chipset.majorVersion != 9 || chipset < kGfx950)
return op->emitOpError("scaled MFMA only supported on gfx908+");
- std::optional<std::tuple<StringRef, uint32_t, uint32_t>>
- maybeScaledIntrinsic = mfmaOpToScaledIntrinsic(op, chipset);
+ std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
+ mfmaOpToScaledIntrinsic(op, chipset);
if (!maybeScaledIntrinsic.has_value())
return op.emitOpError(
"no intrinsic matching scaled MFMA size on given chipset");
@@ -1669,8 +1684,10 @@ struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
/*scales B*/
castScaleOperand(rewriter, loc, adaptor.getScalesB())});
loweredOp.addAttributes(
- {{"cbsz", rewriter.getI32IntegerAttr(aTypeCode)},
- {"blgp", rewriter.getI32IntegerAttr(bTypeCode)},
+ {{"cbsz",
+ ROCDL::MatrixFormatAttr::get(rewriter.getContext(), aTypeCode)},
+ {"blgp",
+ ROCDL::MatrixFormatAttr::get(rewriter.getContext(), bTypeCode)},
{"opselA", rewriter.getI32IntegerAttr(adaptor.getScalesIdxA())},
{"opselB", rewriter.getI32IntegerAttr(adaptor.getScalesIdxB())}});
@@ -2078,8 +2095,10 @@ struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
Type aElemType = getElementTypeOrSelf(op.getSourceA().getType());
Type bElemType = getElementTypeOrSelf(op.getSourceB().getType());
- std::optional<uint32_t> aFmtCode = smallFloatTypeToFormatCode(aElemType);
- std::optional<uint32_t> bFmtCode = smallFloatTypeToFormatCode(bElemType);
+ std::optional<ROCDL::MatrixFormat> aFmtCode =
+ smallFloatTypeToMatrixFormat(aElemType);
+ std::optional<ROCDL::MatrixFormat> bFmtCode =
+ smallFloatTypeToMatrixFormat(bElemType);
if (!aFmtCode || !bFmtCode)
return op.emitOpError("unsupported element types for scaled_wmma");
@@ -2095,8 +2114,10 @@ struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
Type scaleAElemType = scaleAVecType.getElementType();
Type scaleBElemType = scaleBVecType.getElementType();
- std::optional<uint32_t> scaleAFmt = getWmmaScaleFormat(scaleAElemType);
- std::optional<uint32_t> scaleBFmt = getWmmaScaleFormat(scaleBElemType);
+ std::optional<ROCDL::WMMAMatrixScaleFormat> scaleAFmt =
+ getWmmaScaleFormat(scaleAElemType);
+ std::optional<ROCDL::WMMAMatrixScaleFormat> scaleBFmt =
+ getWmmaScaleFormat(scaleBElemType);
if (!scaleAFmt || !scaleBFmt)
return op.emitOpError("unsupported scale element types");
@@ -2114,21 +2135,31 @@ struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
// The f4 variant does not have fmtA and fmtB attributes.
bool is32x16 = (m == 32 && n == 16 && k == 128);
if (!is32x16) {
- attrs.emplace_back("fmtA", rewriter.getI32IntegerAttr(*aFmtCode));
- attrs.emplace_back("fmtB", rewriter.getI32IntegerAttr(*bFmtCode));
+ attrs.emplace_back("fmtA", ROCDL::MatrixFormatAttr::get(
+ rewriter.getContext(), *aFmtCode));
+ attrs.emplace_back("fmtB", ROCDL::MatrixFormatAttr::get(
+ rewriter.getContext(), *bFmtCode));
}
// modC uses default value of 0.
- attrs.emplace_back("modC", rewriter.getI16IntegerAttr(0));
+ attrs.emplace_back(
+ "modC", ROCDL::WMMACModifierAttr::get(rewriter.getContext(),
+ ROCDL::WMMACModifier::none));
// Scale attributes. Convert user-facing firstScaleLane (0 or 16) to the
// half of the wave that is being selected (0 or 1).
- attrs.emplace_back(
- "scaleAType", rewriter.getI32IntegerAttr(op.getAFirstScaleLane() / 16));
- attrs.emplace_back("fmtScaleA", rewriter.getI32IntegerAttr(*scaleAFmt));
- attrs.emplace_back(
- "scaleBType", rewriter.getI32IntegerAttr(op.getBFirstScaleLane() / 16));
- attrs.emplace_back("fmtScaleB", rewriter.getI32IntegerAttr(*scaleBFmt));
+ attrs.emplace_back("scaleAType", ROCDL::WMMAMatrixScaleAttr::get(
+ rewriter.getContext(),
+ static_cast<ROCDL::WMMAMatrixScale>(
+ op.getAFirstScaleLane() / 16)));
+ attrs.emplace_back("fmtScaleA", ROCDL::WMMAMatrixScaleFormatAttr::get(
+ rewriter.getContext(), *scaleAFmt));
+ attrs.emplace_back("scaleBType", ROCDL::WMMAMatrixScaleAttr::get(
+ rewriter.getContext(),
+ static_cast<ROCDL::WMMAMatrixScale>(
+ op.getBFirstScaleLane() / 16)));
+ attrs.emplace_back("fmtScaleB", ROCDL::WMMAMatrixScaleFormatAttr::get(
+ rewriter.getContext(), *scaleBFmt));
// Reuse flags use default value of false.
attrs.emplace_back("reuseA", rewriter.getBoolAttr(false));
@@ -2483,7 +2514,7 @@ struct GlobalLoadAsyncToLDSOpLowering
}
auto offset = rewriter.getI32IntegerAttr(0);
- auto aux = rewriter.getI32IntegerAttr(0);
+ Attribute aux = rewriter.getI32IntegerAttr(0);
switch (transferBits) {
case 8:
@@ -4425,8 +4456,9 @@ struct AMDGPUTensorLoadStoreOpLowering
// will move into the TDM descriptor once it becomes relevant for future use
auto v8i32 = VectorType::get(8, rewriter.getI32Type());
Value dgroup4 = LLVM::ZeroOp::create(rewriter, op.getLoc(), v8i32);
+ Attribute cachePolicy = rewriter.getI32IntegerAttr(0);
rewriter.replaceOpWithNewOp<TargetOp>(op, desc[0], desc[1], desc[2],
- desc[3], dgroup4, /*cachePolicy=*/0,
+ desc[3], dgroup4, cachePolicy,
/*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr,
/*tbaa=*/nullptr);
@@ -4448,7 +4480,11 @@ struct GlobalPrefetchOpLowering
const bool isSpeculative = op.getSpeculative();
const int32_t immArgValue = getGlobalPrefetchLLVMEncoding(
op.getTemporalHint(), op.getCacheScope(), isSpeculative);
- IntegerAttr immArgAttr = rewriter.getI32IntegerAttr(immArgValue);
+ // amdgpu.global_prefetch is gfx1250+, so its policy bits use gfx12
+ // encoding.
+ Attribute cachePolicy = ROCDL::Gfx12CachePolicyAttr::get(
+ rewriter.getContext(),
+ static_cast<ROCDL::Gfx12CachePolicy>(immArgValue));
ValueRange indices = adaptor.getIndices();
Value memRef = adaptor.getSrc();
@@ -4462,7 +4498,7 @@ struct GlobalPrefetchOpLowering
rewriter, loc, memRefType, descriptor, indices, inboundsFlags);
rewriter.replaceOpWithNewOp<ROCDL::GlobalPrefetchOp>(
- op, prefetchPtr, immArgAttr, mlir::ArrayAttr{}, mlir::ArrayAttr{},
+ op, prefetchPtr, cachePolicy, mlir::ArrayAttr{}, mlir::ArrayAttr{},
mlir::ArrayAttr{});
return success();
}
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUOps.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUOps.cpp
index bbd98fb0a0117..ad0ff12c9307f 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUOps.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUOps.cpp
@@ -563,7 +563,7 @@ LogicalResult MFMAOp::verify() {
" result values for this operation but got " +
Twine(destLen));
- if (destElem.isF64() && getBlgp() != MFMAPermB::none)
+ if (destElem.isF64() && getBlgp() != ROCDL::MFMAPermB::none)
return emitOpError(
"double-precision ops do not support permuting lanes of B");
if (destElem.isF64() && getCbsz() != 0)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 29d30047f8daa..6e69a07856300 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -24,162 +24,16 @@
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/Operation.h"
#include "mlir/Transforms/InliningUtils.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
using namespace mlir;
using namespace ROCDL;
#include "mlir/Dialect/LLVMIR/ROCDLOpsDialect.cpp.inc"
-
-//===----------------------------------------------------------------------===//
-// Parsing for ROCDL ops
-//===----------------------------------------------------------------------===//
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.load.* %rsrc, %offset, %soffset, %aux
-// : result_type`
-ParseResult RawBufferLoadOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 4> ops;
- Type type;
- if (parser.parseOperandList(ops, 4) || parser.parseColonType(type) ||
- parser.addTypeToList(type, result.types))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
- return parser.resolveOperands(ops, {i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands);
-}
-
-void RawBufferLoadOp::print(OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getRes().getType();
-}
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.store.* %vdata, %rsrc, %offset,
-// %soffset, %aux : result_type`
-ParseResult RawBufferStoreOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 5> ops;
- Type type;
- if (parser.parseOperandList(ops, 5) || parser.parseColonType(type))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
-
- if (parser.resolveOperands(ops, {type, i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands))
- return failure();
- return success();
-}
-
-void RawBufferStoreOp::print(OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getVdata().getType();
-}
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.atomic.fadd.* %vdata, %rsrc, %offset,
-// %soffset, %aux : result_type`
-ParseResult RawBufferAtomicFAddOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 5> ops;
- Type type;
- if (parser.parseOperandList(ops, 5) || parser.parseColonType(type) ||
- parser.addTypeToList(type, result.types))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
-
- if (parser.resolveOperands(ops, {type, i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands))
- return failure();
- return success();
-}
-
-void RawBufferAtomicFAddOp::print(mlir::OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getVdata().getType();
-}
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.atomic.fmax.* %vdata, %rsrc, %offset,
-// %soffset, %aux : result_type`
-ParseResult RawBufferAtomicFMaxOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 5> ops;
- Type type;
- if (parser.parseOperandList(ops, 5) || parser.parseColonType(type) ||
- parser.addTypeToList(type, result.types))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
-
- if (parser.resolveOperands(ops, {type, i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands))
- return failure();
- return success();
-}
-
-void RawBufferAtomicFMaxOp::print(mlir::OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getVdata().getType();
-}
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.atomic.smax.* %vdata, %rsrc, %offset,
-// %soffset, %aux : result_type`
-ParseResult RawBufferAtomicSMaxOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 5> ops;
- Type type;
- if (parser.parseOperandList(ops, 5) || parser.parseColonType(type) ||
- parser.addTypeToList(type, result.types))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
-
- if (parser.resolveOperands(ops, {type, i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands))
- return failure();
- return success();
-}
-
-void RawBufferAtomicSMaxOp::print(mlir::OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getVdata().getType();
-}
-
-// <operation> ::=
-// `llvm.amdgcn.raw.buffer.atomic.umin.* %vdata, %rsrc, %offset,
-// %soffset, %aux : result_type`
-ParseResult RawBufferAtomicUMinOp::parse(OpAsmParser &parser,
- OperationState &result) {
- SmallVector<OpAsmParser::UnresolvedOperand, 5> ops;
- Type type;
- if (parser.parseOperandList(ops, 5) || parser.parseColonType(type) ||
- parser.addTypeToList(type, result.types))
- return failure();
-
- auto bldr = parser.getBuilder();
- auto int32Ty = bldr.getI32Type();
- auto i32x4Ty = VectorType::get({4}, int32Ty);
-
- if (parser.resolveOperands(ops, {type, i32x4Ty, int32Ty, int32Ty, int32Ty},
- parser.getNameLoc(), result.operands))
- return failure();
- return success();
-}
-
-void RawBufferAtomicUMinOp::print(mlir::OpAsmPrinter &p) {
- p << " " << getOperands() << " : " << getVdata().getType();
-}
+#include "mlir/Dialect/LLVMIR/ROCDLOpsEnums.cpp.inc"
//===----------------------------------------------------------------------===//
// ROCDLDialect initialization, type parsing, and registration.
@@ -224,6 +78,86 @@ LogicalResult ROCDLDialect::verifyOperationAttribute(Operation *op,
return success();
}
+//===----------------------------------------------------------------------===//
+// ROCDL op custom parsers/printers.
+//===----------------------------------------------------------------------===//
+
+template <typename EnumAttrT, typename EnumT>
+static ParseResult parseCachePolicyEnum(OpAsmParser &parser,
+ Attribute &cachePolicy) {
+ if (parser.parseLess())
+ return failure();
+ FailureOr<EnumT> parsed = FieldParser<EnumT>::parse(parser);
+ if (failed(parsed))
+ return failure();
+ if (parser.parseGreater())
+ return failure();
+ cachePolicy = EnumAttrT::get(parser.getContext(), *parsed);
+ return success();
+}
+
+static ParseResult parseCachePolicy(OpAsmParser &parser,
+ Attribute &cachePolicy) {
+ uint32_t rawValue;
+ OptionalParseResult rawValueParseResult =
+ parser.parseOptionalInteger(rawValue);
+ if (rawValueParseResult.has_value()) {
+ if (failed(*rawValueParseResult))
+ return failure();
+ cachePolicy =
+ IntegerAttr::get(IntegerType::get(parser.getContext(), 32), rawValue);
+ return success();
+ }
+
+ StringRef policyFamily;
+ auto loc = parser.getCurrentLocation();
+ if (failed(parser.parseOptionalKeyword(
+ &policyFamily, {"pre_gfx12", "gfx942", "gfx12", "gfx12_atomic"}))) {
+ return parser.emitError(loc)
+ << "expected cache policy family 'pre_gfx12', 'gfx942', 'gfx12', "
+ "'gfx12_atomic', or a 32-bit integer";
+ }
+
+ if (policyFamily == "pre_gfx12")
+ return parseCachePolicyEnum<PreGfx12CachePolicyAttr, PreGfx12CachePolicy>(
+ parser, cachePolicy);
+ if (policyFamily == "gfx942")
+ return parseCachePolicyEnum<Gfx942CachePolicyAttr, Gfx942CachePolicy>(
+ parser, cachePolicy);
+ if (policyFamily == "gfx12")
+ return parseCachePolicyEnum<Gfx12CachePolicyAttr, Gfx12CachePolicy>(
+ parser, cachePolicy);
+ return parseCachePolicyEnum<Gfx12AtomicCachePolicyAttr,
+ Gfx12AtomicCachePolicy>(parser, cachePolicy);
+}
+
+template <typename EnumAttrT>
+static void printCachePolicyEnum(OpAsmPrinter &printer, EnumAttrT cachePolicy,
+ StringRef family) {
+ printer << family << "<" << cachePolicy.getValue() << ">";
+}
+
+static void printCachePolicy(OpAsmPrinter &printer, Operation *,
+ Attribute cachePolicy) {
+ llvm::TypeSwitch<Attribute>(cachePolicy)
+ .Case<IntegerAttr>([&](IntegerAttr rawPolicy) {
+ printer << rawPolicy.getValue().getZExtValue();
+ })
+ .Case<PreGfx12CachePolicyAttr>([&](PreGfx12CachePolicyAttr policy) {
+ printCachePolicyEnum(printer, policy, "pre_gfx12");
+ })
+ .Case<Gfx942CachePolicyAttr>([&](Gfx942CachePolicyAttr policy) {
+ printCachePolicyEnum(printer, policy, "gfx942");
+ })
+ .Case<Gfx12CachePolicyAttr>([&](Gfx12CachePolicyAttr policy) {
+ printCachePolicyEnum(printer, policy, "gfx12");
+ })
+ .Case<Gfx12AtomicCachePolicyAttr>([&](Gfx12AtomicCachePolicyAttr policy) {
+ printCachePolicyEnum(printer, policy, "gfx12_atomic");
+ })
+ .DefaultUnreachable("unknown ROCDL cache policy attribute");
+}
+
//===----------------------------------------------------------------------===//
// ROCDL target attribute.
//===----------------------------------------------------------------------===//
@@ -249,7 +183,7 @@ ROCDLTargetAttr::verify(function_ref<InFlightDiagnostic()> emitError,
return failure();
}
if (files && !llvm::all_of(files, [](::mlir::Attribute attr) {
- return attr && mlir::isa<StringAttr>(attr);
+ return mlir::isa_and_nonnull<StringAttr>(attr);
})) {
emitError() << "All the elements in the `link` array must be strings.";
return failure();
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 8b1c2bcf6e1d7..01d3e9c4a62ce 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -981,11 +981,16 @@ llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size());
for (auto [immArgPos, immArgName] :
llvm::zip(immArgPositions, immArgAttrNames)) {
- auto attr = llvm::cast<TypedAttr>(intrOp->getAttr(immArgName));
- assert(attr.getType().isIntOrFloat() && "expected int or float immarg");
- auto *type = moduleTranslation.convertType(attr.getType());
+ Attribute attr = intrOp->getAttr(immArgName);
+ if (auto intrinsicIntegerAttr =
+ dyn_cast<LLVM::IntrinsicIntegerAttrInterface>(attr))
+ attr = intrinsicIntegerAttr.getIntegerAttr();
+ auto typedAttr = llvm::cast<TypedAttr>(attr);
+ assert(typedAttr.getType().isIntOrFloat() &&
+ "expected int or float immarg");
+ auto *type = moduleTranslation.convertType(typedAttr.getType());
args[immArgPos] = LLVM::detail::getLLVMConstant(
- type, attr, intrOp->getLoc(), moduleTranslation);
+ type, typedAttr, intrOp->getLoc(), moduleTranslation);
}
unsigned opArg = 0;
for (auto &arg : args) {
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index 9131ac05ebe30..f6c0241a89484 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -130,7 +130,7 @@ func.func @gpu_gcn_raw_buffer_load_scalar_i32(%buf: memref<i32>) -> i32 {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %[[stride]], %[[numRecords]], %[[flags]] : !llvm.ptr to <8>
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[] : memref<i32> -> i32
func.return %0 : i32
@@ -144,7 +144,7 @@ func.func @gpu_gcn_raw_buffer_load_i32(%buf: memref<64xi32>, %idx: i32) -> i32 {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %[[stride]], %[[numRecords]], %[[flags]] : !llvm.ptr to <8>
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> i32
func.return %0 : i32
@@ -178,8 +178,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_strided(%buf: memref<16x16xi32, strided<[
// CHECK: %[[vgpr_off:.*]] = llvm.mul %[[index]], %[[elem_size]] : i32
// CHECK: %[[zero_0:.*]] = llvm.mlir.constant(0 : i32) : i32
// CHECK: %[[sgpr_off:.*]] = llvm.mul %[[zero_0]], %[[elem_size]] : i32
- // CHECK: %[[zero_1:.*]] = llvm.mlir.constant(0 : i32) : i32
- // CHECK: %[[v:.*]] = rocdl.raw.ptr.buffer.load %[[rsrc]], %[[vgpr_off]], %[[sgpr_off]], %[[zero_1]] : i32
+ // CHECK: %[[v:.*]] = rocdl.raw.ptr.buffer.load %[[rsrc]], %[[vgpr_off]], %[[sgpr_off]], 0 : i32
// CHECK: return %[[v]] : i32
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%i, %j] : memref<16x16xi32, strided<[?, ?], offset: ?>>, i32, i32 -> i32
func.return %0 : i32
@@ -191,7 +190,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_oob_off(%buf: memref<64xi32>, %idx: i32)
// RDNA: %[[flags:.*]] = llvm.mlir.constant(553807872 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %{{.*}}, %[[flags]]
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = false} %buf[%idx] : memref<64xi32>, i32 -> i32
func.return %0 : i32
@@ -199,7 +198,7 @@ func.func @gpu_gcn_raw_buffer_load_i32_oob_off(%buf: memref<64xi32>, %idx: i32)
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_1xi32
func.func @gpu_gcn_raw_buffer_load_1xi32(%buf: memref<64xi32>, %idx: i32) -> vector<1xi32> {
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
// CHECK: %[[cast:.*]] = llvm.bitcast %[[ret]] : i32 to vector<1xi32>
// CHECK: return %[[cast]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> vector<1xi32>
@@ -208,7 +207,7 @@ func.func @gpu_gcn_raw_buffer_load_1xi32(%buf: memref<64xi32>, %idx: i32) -> vec
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_2xi32
func.func @gpu_gcn_raw_buffer_load_2xi32(%buf: memref<64xi32>, %idx: i32) -> vector<2xi32> {
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<2xi32>
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xi32>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi32>, i32 -> vector<2xi32>
func.return %0 : vector<2xi32>
@@ -218,7 +217,7 @@ func.func @gpu_gcn_raw_buffer_load_2xi32(%buf: memref<64xi32>, %idx: i32) -> vec
func.func @gpu_gcn_raw_buffer_load_i8(%buf: memref<64xi8>, %idx: i32) -> i8 {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i8
+ // CHECK: %[[ret:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i8
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> i8
func.return %0 : i8
@@ -228,7 +227,7 @@ func.func @gpu_gcn_raw_buffer_load_i8(%buf: memref<64xi8>, %idx: i32) -> i8 {
func.func @gpu_gcn_raw_buffer_load_2xi8(%buf: memref<64xi8>, %idx: i32) -> vector<2xi8> {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i16
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i16
// CHECK: %[[ret:.*]] = llvm.bitcast %[[loaded]] : i16 to vector<2xi8>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> vector<2xi8>
@@ -237,7 +236,7 @@ func.func @gpu_gcn_raw_buffer_load_2xi8(%buf: memref<64xi8>, %idx: i32) -> vecto
// CHECK-LABEL: func @gpu_gcn_raw_buffer_load_16xi8
func.func @gpu_gcn_raw_buffer_load_16xi8(%buf: memref<64xi8>, %idx: i32) -> vector<16xi8> {
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<4xi32>
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>
// CHECK: %[[ret:.*]] = llvm.bitcast %[[loaded]] : vector<4xi32> to vector<16xi8>
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xi8>, i32 -> vector<16xi8>
@@ -248,7 +247,7 @@ func.func @gpu_gcn_raw_buffer_load_16xi8(%buf: memref<64xi8>, %idx: i32) -> vect
func.func @gpu_gcn_raw_buffer_load_f8E5M2FNUZ(%buf: memref<64xf8E5M2FNUZ>, %idx: i32) -> f8E5M2FNUZ {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i8
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i8
// CHECK: %[[ret:.*]] = builtin.unrealized_conversion_cast %[[loaded]] : i8 to f8E5M2FNUZ
// CHECK: return %[[ret]]
%0 = amdgpu.raw_buffer_load {boundsCheck = true} %buf[%idx] : memref<64xf8E5M2FNUZ>, i32 -> f8E5M2FNUZ
@@ -259,7 +258,7 @@ func.func @gpu_gcn_raw_buffer_load_f8E5M2FNUZ(%buf: memref<64xf8E5M2FNUZ>, %idx:
func.func @gpu_gcn_raw_buffer_load_4xf8E4M3FNUZ(%buf: memref<64xf8E4M3FNUZ>, %idx: i32) -> vector<4xf8E4M3FNUZ> {
// CHECK: %[[numRecords:.*]] = llvm.mlir.constant(64 : i64)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %{{.*}}
- // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[loaded:.*]] = rocdl.raw.ptr.buffer.load %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: %[[cast:.*]] = llvm.bitcast %[[loaded]] : i32 to vector<4xi8>
// CHECK: %[[ret:.*]] = builtin.unrealized_conversion_cast %[[cast]] : vector<4xi8> to vector<4xf8E4M3FNUZ>
// CHECK: return %[[ret]]
@@ -274,7 +273,7 @@ func.func @gpu_gcn_raw_buffer_store_scalar_i32(%value: i32, %buf: memref<i32>) {
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %{{.*}}, %[[flags]]
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[] : i32 -> memref<i32>
func.return
}
@@ -286,7 +285,7 @@ func.func @gpu_gcn_raw_buffer_store_i32(%value: i32, %buf: memref<64xi32>, %idx:
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return
}
@@ -294,7 +293,7 @@ func.func @gpu_gcn_raw_buffer_store_i32(%value: i32, %buf: memref<64xi32>, %idx:
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_1xf32
func.func @gpu_gcn_raw_buffer_store_1xf32(%value: vector<1xf32>, %buf: memref<64xf32>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<1xf32> to f32
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<1xf32> -> memref<64xf32>, i32
func.return
}
@@ -302,7 +301,7 @@ func.func @gpu_gcn_raw_buffer_store_1xf32(%value: vector<1xf32>, %buf: memref<64
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_2xi8
func.func @gpu_gcn_raw_buffer_store_2xi8(%value: vector<2xi8>, %buf: memref<64xi8>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<2xi8> to i16
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i16
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : i16
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<2xi8> -> memref<64xi8>, i32
func.return
}
@@ -310,7 +309,7 @@ func.func @gpu_gcn_raw_buffer_store_2xi8(%value: vector<2xi8>, %buf: memref<64xi
// CHECK-LABEL: func @gpu_gcn_raw_buffer_store_16xi8
func.func @gpu_gcn_raw_buffer_store_16xi8(%value: vector<16xi8>, %buf: memref<64xi8>, %idx: i32) {
// CHECK: %[[cast:.*]] = llvm.bitcast %{{.*}} : vector<16xi8> to vector<4xi32>
- // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<4xi32>
+ // CHECK: rocdl.raw.ptr.buffer.store %[[cast]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>
amdgpu.raw_buffer_store {boundsCheck = true} %value -> %buf[%idx] : vector<16xi8> -> memref<64xi8>, i32
func.return
}
@@ -323,7 +322,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_f32(%value: f32, %buf: memref<64xf32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : f32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : f32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : f32 -> memref<64xf32>, i32
func.return %old : f32
@@ -336,7 +335,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_v2f16(%value: vector<2xf16>, %buf: mem
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : vector<2xf16>
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : vector<2xf16>
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : vector<2xf16> -> memref<64xf16>, i32
func.return %old : vector<2xf16>
@@ -349,7 +348,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fadd_v2bf16(%value: vector<2xbf16>, %buf: m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : vector<2xbf16>
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : vector<2xbf16>
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fadd {boundsCheck = true} %value -> %buf[%idx] : vector<2xbf16> -> memref<64xbf16>, i32
func.return %old : vector<2xbf16>
@@ -362,7 +361,7 @@ func.func @gpu_gcn_raw_buffer_atomic_fmax_f32(%value: f32, %buf: memref<64xf32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : f32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : f32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_fmax {boundsCheck = true} %value -> %buf[%idx] : f32 -> memref<64xf32>, i32
func.return %old : f32
@@ -375,7 +374,7 @@ func.func @gpu_gcn_raw_buffer_atomic_smax_i32(%value: i32, %buf: memref<64xi32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_smax {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return %old : i32
@@ -388,7 +387,7 @@ func.func @gpu_gcn_raw_buffer_atomic_umin_i32(%value: i32, %buf: memref<64xi32>,
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[old:.*]] = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: return %[[old]]
%old = amdgpu.raw_buffer_atomic_umin {boundsCheck = true} %value -> %buf[%idx] : i32 -> memref<64xi32>, i32
func.return %old : i32
@@ -404,7 +403,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %buf : m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcCast]], %[[cmpCast]], %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcCast]], %[[cmpCast]], %[[resource]], %{{.*}}, %{{.*}}, 0 : i32
// CHECK: %[[dstCast:.*]] = llvm.bitcast %[[dst]] : i32 to f32
// CHECK: return %[[dstCast]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : f32 -> memref<64xf32>, i32
@@ -419,7 +418,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_i64(%src : i64, %cmp : i64, %buf : m
// RDNA: %[[flags:.*]] = llvm.mlir.constant(822243328 : i32)
// GFX1250: %[[flags:.*]] = llvm.mlir.constant(0 : i32)
// CHECK: %[[resource:.*]] = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %[[numRecords]], %[[flags]]
- // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[src]], %[[cmp]], %[[resource]], %{{.*}}, %{{.*}}, %{{.*}} : i64
+ // CHECK: %[[dst:.*]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[src]], %[[cmp]], %[[resource]], %{{.*}}, %{{.*}}, 0 : i64
// CHECK: return %[[dst]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : i64 -> memref<64xi64>, i32
func.return %dst : i64
@@ -430,7 +429,7 @@ func.func @amdgpu_raw_buffer_atomic_cmpswap_i64(%src : i64, %cmp : i64, %buf : m
func.func @amdgpu_raw_buffer_atomic_cmpswap_v2f16(%src : vector<2xf16>, %cmp : vector<2xf16>, %buf : memref<64xf16>, %idx: i32) -> vector<2xf16> {
// CHECK-DAG: %[[srcBits:.+]] = llvm.bitcast %[[src]] : vector<2xf16> to i32
// CHECK-DAG: %[[cmpBits:.+]] = llvm.bitcast %[[cmp]] : vector<2xf16> to i32
- // CHECK: %[[dstBits:.+]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcBits]], %[[cmpBits]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %[[dstBits:.+]] = rocdl.raw.ptr.buffer.atomic.cmpswap %[[srcBits]], %[[cmpBits]], %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
// CHECK: %[[dst:.+]] = llvm.bitcast %[[dstBits]] : i32 to vector<2xf16>
// CHECK: return %[[dst]]
%dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : vector<2xf16> -> memref<64xf16>, i32
@@ -457,31 +456,13 @@ func.func @lds_barrier() {
// CHECK-LABEL: func @sched_barrier
func.func @sched_barrier() {
- // CHECK: rocdl.sched.barrier 0
- amdgpu.sched_barrier allow = <none>
- // CHECK: rocdl.sched.barrier 1
- amdgpu.sched_barrier allow = <non_mem_non_sideffect>
- // CHECK: rocdl.sched.barrier 2
- amdgpu.sched_barrier allow = <valu>
- // CHECK: rocdl.sched.barrier 4
- amdgpu.sched_barrier allow = <salu>
- // CHECK: rocdl.sched.barrier 8
- amdgpu.sched_barrier allow = <mfma_wmma>
- // CHECK: rocdl.sched.barrier 16
- amdgpu.sched_barrier allow = <all_vmem>
- // CHECK: rocdl.sched.barrier 32
- amdgpu.sched_barrier allow = <vmem_read>
- // CHECK: rocdl.sched.barrier 64
- amdgpu.sched_barrier allow = <vmem_write>
- // CHECK: rocdl.sched.barrier 128
- amdgpu.sched_barrier allow = <all_ds>
- // CHECK: rocdl.sched.barrier 256
- amdgpu.sched_barrier allow = <ds_read>
- // CHECK: rocdl.sched.barrier 512
- amdgpu.sched_barrier allow = <ds_write>
- // CHECK: rocdl.sched.barrier 1024
- amdgpu.sched_barrier allow = <transcendental>
- // CHECK: rocdl.sched.barrier 18
- amdgpu.sched_barrier allow = <valu|all_vmem>
+ // CHECK: rocdl.sched.barrier none
+ amdgpu.sched_barrier allow = none
+ // CHECK: rocdl.sched.barrier non_mem_non_sideeffect
+ amdgpu.sched_barrier allow = non_mem_non_sideeffect
+ // CHECK: rocdl.sched.barrier valu|all_vmem
+ amdgpu.sched_barrier allow = valu|all_vmem
+ // CHECK: rocdl.sched.barrier all
+ amdgpu.sched_barrier allow = all
func.return
}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index e43ece8c74fdf..bc67513ff02a6 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -728,7 +728,7 @@ func.func @make_dma_descriptor_workgroup_mask(%base: !amdgpu.tdm_base<i32>, %wg_
func.func @tensor_load_to_lds(%desc: !amdgpu.tdm_descriptor) {
// CHECK: %[[DGROUPS:.+]]:4 = builtin.unrealized_conversion_cast %[[DESC]]
// CHECK: %[[DGROUP4:.+]] = llvm.mlir.zero : vector<8xi32>
- // CHECK: rocdl.tensor.load.to.lds %[[DGROUPS]]#0, %[[DGROUPS]]#1, %[[DGROUPS]]#2, %[[DGROUPS]]#3, %[[DGROUP4]] cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ // CHECK: rocdl.tensor.load.to.lds %[[DGROUPS]]#0, %[[DGROUPS]]#1, %[[DGROUPS]]#2, %[[DGROUPS]]#3, %[[DGROUP4]], 0 : vector<4xi32>, vector<8xi32>
amdgpu.tensor_load_to_lds %desc : !amdgpu.tdm_descriptor
func.return
}
@@ -738,7 +738,7 @@ func.func @tensor_load_to_lds(%desc: !amdgpu.tdm_descriptor) {
func.func @tensor_store_from_lds(%desc: !amdgpu.tdm_descriptor) {
// CHECK: %[[DGROUPS:.+]]:4 = builtin.unrealized_conversion_cast %[[DESC]]
// CHECK: %[[DGROUP4:.+]] = llvm.mlir.zero : vector<8xi32>
- // CHECK: rocdl.tensor.store.from.lds %[[DGROUPS]]#0, %[[DGROUPS]]#1, %[[DGROUPS]]#2, %[[DGROUPS]]#3, %[[DGROUP4]] cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ // CHECK: rocdl.tensor.store.from.lds %[[DGROUPS]]#0, %[[DGROUPS]]#1, %[[DGROUPS]]#2, %[[DGROUPS]]#3, %[[DGROUP4]], 0 : vector<4xi32>, vector<8xi32>
amdgpu.tensor_store_from_lds %desc : !amdgpu.tdm_descriptor
func.return
}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir b/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
index acd3710a485ac..f71de64cd071f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/global-prefetch.mlir
@@ -3,7 +3,7 @@
// CHECK-LABEL: @glb_prefetch0
func.func @glb_prefetch0(%src : memref<64x64xf16, #gpu.address_space<global>>, %i : i64, %j : i64) {
// CHECK: %[[PTR:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
- // CHECK: rocdl.global.prefetch %[[PTR]], scope 3 : !llvm.ptr<1>
+ // CHECK: rocdl.global.prefetch %[[PTR]], gfx12<lu> : !llvm.ptr<1>
amdgpu.global_prefetch %src[%i, %j] HT WGP : memref<64x64xf16, #gpu.address_space<global>>
func.return
}
@@ -11,7 +11,7 @@ func.func @glb_prefetch0(%src : memref<64x64xf16, #gpu.address_space<global>>, %
// CHECK-LABEL: @glb_prefetch1
func.func @glb_prefetch1(%src : memref<64x64xf16, #gpu.address_space<global>>, %i : i64, %j : i64) {
// CHECK: %[[PTR:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
- // CHECK: rocdl.global.prefetch %[[PTR]], scope 10 : !llvm.ptr<1>
+ // CHECK: rocdl.global.prefetch %[[PTR]], gfx12<ht|scope_se> : !llvm.ptr<1>
amdgpu.global_prefetch %src[%i, %j] HT SE speculative : memref<64x64xf16, #gpu.address_space<global>>
func.return
}
@@ -19,7 +19,15 @@ func.func @glb_prefetch1(%src : memref<64x64xf16, #gpu.address_space<global>>, %
// CHECK-LABEL: @glb_prefetch2
func.func @glb_prefetch2(%src : memref<64x64xf16, #gpu.address_space<global>>, %i : i64, %j : i64) {
// CHECK: %[[PTR:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
- // CHECK: rocdl.global.prefetch %{{.*}}, scope 16 : !llvm.ptr<1>
+ // CHECK: rocdl.global.prefetch %{{.*}}, gfx12<scope_dev> : !llvm.ptr<1>
amdgpu.global_prefetch %src[%i, %j] RT DEV speculative : memref<64x64xf16, #gpu.address_space<global>>
func.return
}
+
+// CHECK-LABEL: @glb_prefetch3
+func.func @glb_prefetch3(%src : memref<64x64xf16, #gpu.address_space<global>>, %i : i64, %j : i64) {
+ // CHECK: %[[PTR:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ // CHECK: rocdl.global.prefetch %[[PTR]], gfx12<nt_rt|scope_se> : !llvm.ptr<1>
+ amdgpu.global_prefetch %src[%i, %j] NT_RT SE speculative : memref<64x64xf16, #gpu.address_space<global>>
+ func.return
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
index 39d90ceb14436..4812be43fff8f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
@@ -20,30 +20,30 @@ func.func @mfma_to_rocdl(%arg0 : vector<8xf16>, %arg1 : vector<16xf32>,
amdgpu.mfma 16x16x64 %arg4 * %arg4 + %arg6 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<16xi8>, vector<16xi8>, vector<4xi32>
// CHECK: %[[c0:.+]] = llvm.mlir.constant(0 : i32) : i32
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg7 * %arg7 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 0, 0, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e4m3, fp8_e4m3, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg7 * %arg7 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<4xf32>
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 1, 1, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e5m2, fp8_e5m2, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg8 * %arg8 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 1, 1, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp8_e5m2, fp8_e5m2, 0, %[[c0]], 0, %[[c0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg8 * %arg8 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<4xf32>
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 2, 2, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e2m3, fp6_e2m3, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg9 * %arg9 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E2M3FN>, vector<32xf6E2M3FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 2, 2, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e2m3, fp6_e2m3, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg9 * %arg9 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E2M3FN>, vector<32xf6E2M3FN>, vector<4xf32>
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 3, 3, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e3m2, fp6_e3m2, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg10 * %arg10 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E3M2FN>, vector<32xf6E3M2FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 3, 3, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e3m2, fp6_e3m2, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg10 * %arg10 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E3M2FN>, vector<32xf6E3M2FN>, vector<4xf32>
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 4, 4, 0, %[[c0]], 0, %[[c0]] : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp4_e2m1, fp4_e2m1, 0, %[[c0]], 0, %[[c0]] : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg11 * %arg11 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf4E2M1FN>, vector<32xf4E2M1FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 4, 4, 0, %[[c0]], 0, %[[c0]] : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp4_e2m1, fp4_e2m1, 0, %[[c0]], 0, %[[c0]] : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg11 * %arg11 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf4E2M1FN>, vector<32xf4E2M1FN>, vector<4xf32>
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 2, 4, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e2m3, fp4_e2m1, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.mfma 32x32x64 %arg9 * %arg11 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E2M3FN>, vector<32xf4E2M1FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, 2, 4, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, fp6_e2m3, fp4_e2m1, 0, %[[c0]], 0, %[[c0]] : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.mfma 16x16x128 %arg9 * %arg11 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<32xf6E2M3FN>, vector<32xf4E2M1FN>, vector<4xf32>
func.return
@@ -61,37 +61,37 @@ func.func @scaled_mfma_to_rocdl(%arg0 : vector<16xf32>,
// CHECK: %[[b0:.+]] = llvm.bitcast {{.*}} : vector<4xi8> to i32
// CHECK: %[[z0:.+]] = llvm.zext {{.*}} : i8 to i32
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 0, 0, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp8_e4m3, fp8_e4m3, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.scaled_mfma 32x32x64 (%arg7[0] * %arg2) * (%arg8[1] * %arg2) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf8E4M3FN>, f8E8M0FNU, vector<32xf8E4M3FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 0, 0, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp8_e4m3, fp8_e4m3, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.scaled_mfma 16x16x128 (%arg7[0] * %arg2) * (%arg8[1] * %arg2) + %arg1 : vector<4xf8E8M0FNU>, vector<32xf8E4M3FN>, f8E8M0FNU, vector<32xf8E4M3FN>, vector<4xf32>
// CHECK: llvm.bitcast
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 1, 1, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp8_e5m2, fp8_e5m2, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.scaled_mfma 32x32x64 (%arg7[0] * %arg3) * (%arg8[1] * %arg3) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf8E5M2>, f8E8M0FNU, vector<32xf8E5M2>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 1, 1, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp8_e5m2, fp8_e5m2, 0, %[[b0]], 1, %[[z0]] : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.scaled_mfma 16x16x128 (%arg7[0] * %arg3) * (%arg8[1] * %arg3) + %arg1 : vector<4xf8E8M0FNU>, vector<32xf8E5M2>, f8E8M0FNU, vector<32xf8E5M2>, vector<4xf32>
// CHECK: llvm.bitcast
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 2, 2, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp6_e2m3, fp6_e2m3, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.scaled_mfma 32x32x64 (%arg7[0] * %arg4) * (%arg8[1] * %arg4) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 2, 2, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp6_e2m3, fp6_e2m3, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.scaled_mfma 16x16x128 (%arg7[0] * %arg4) * (%arg8[1] * %arg4) + %arg1 : vector<4xf8E8M0FNU>, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<4xf32>
// CHECK: llvm.bitcast
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 3, 3, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp6_e3m2, fp6_e3m2, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.scaled_mfma 32x32x64 (%arg7[0] * %arg5) * (%arg8[1] * %arg5) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf6E3M2FN>, f8E8M0FNU, vector<32xf6E3M2FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 3, 3, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp6_e3m2, fp6_e3m2, 0, %[[b0]], 1, %[[z0]] : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.scaled_mfma 16x16x128 (%arg7[0] * %arg5) * (%arg8[1] * %arg5) + %arg1 : vector<4xf8E8M0FNU>, vector<32xf6E3M2FN>, f8E8M0FNU, vector<32xf6E3M2FN>, vector<4xf32>
// CHECK: llvm.bitcast
- // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 4, 4, 0, %[[b0]], 1, %[[z0]] : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp4_e2m1, fp4_e2m1, 0, %[[b0]], 1, %[[z0]] : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
amdgpu.scaled_mfma 32x32x64 (%arg7[0] * %arg6) * (%arg8[1] * %arg6) + %arg0 : vector<4xf8E8M0FNU>, vector<32xf4E2M1FN>, f8E8M0FNU, vector<32xf4E2M1FN>, vector<16xf32>
- // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, 4, 4, 0, %[[b0]], 1, %[[z0]] : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, {{.*}}, fp4_e2m1, fp4_e2m1, 0, %[[b0]], 1, %[[z0]] : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
amdgpu.scaled_mfma 16x16x128 (%arg7[0] * %arg6) * (%arg8[1] * %arg6) + %arg1 : vector<4xf8E8M0FNU>, vector<32xf4E2M1FN>, f8E8M0FNU, vector<32xf4E2M1FN>, vector<4xf32>
func.return
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
index 8d2f38794de5b..0f8917841929f 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma.mlir
@@ -61,8 +61,8 @@ func.func @mfma_to_rocdl(%arg0 : f32, %arg1 : vector<32xf32>,
amdgpu.mfma 32x32x8 %arg10 * %arg10 + %arg2 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<4xbf16>, vector<4xbf16>, vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x16bf16.1k{{.*}}: (vector<4xi16>, vector<4xi16>, vector<4xf32>) -> vector<4xf32>
amdgpu.mfma 16x16x16 %arg10 * %arg10 + %arg3 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<4xbf16>, vector<4xbf16>, vector<4xf32>
- // CHECK: rocdl.mfma.f64.16x16x4f64{{.*}}: (f64, f64, vector<4xf64>) -> vector<4xf64>
- amdgpu.mfma 16x16x4 %arg11 * %arg11 + %arg12 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : f64, f64, vector<4xf64>
+ // CHECK: rocdl.mfma.f64.16x16x4f64{{.*}}, 0, 0, neg_a|neg_b : (f64, f64, vector<4xf64>) -> vector<4xf64>
+ amdgpu.mfma 16x16x4 %arg11 * %arg11 + %arg12 { abid = 0 : i32, cbsz = 0 : i32, negateA, negateB } blgp = none : f64, f64, vector<4xf64>
// CHECK: rocdl.mfma.f64.4x4x4f64{{.*}}: (f64, f64, f64) -> f64
amdgpu.mfma 4x4x4 %arg11 * %arg11 + %arg11 { abid = 0 : i32, cbsz = 0 : i32, blocks = 4 : i32 } blgp = none : f64, f64, f64
// CHECK: %[[BITCAST_8xi8_i64:.+]] = llvm.bitcast {{.*}} : vector<8xi8> to i64
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
index 5fabd0797ee79..429e05e46275d 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/wmma-gfx1250.mlir
@@ -35,25 +35,25 @@ func.func @wmma_k64(%arg0 : vector<32xi8>, %arg1 : vector<32xf8E4M3FN>, %arg2 :
// CHECK: rocdl.wmma.f32.16x16x64.fp8_fp8 {{.*}}, {{.*}}, %arg4
amdgpu.wmma 16x16x64 %arg1 * %arg1 + %arg4 : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x64.fp8_fp8 {{.*}}, {{.*}}, %arg5 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x64.fp8_fp8 {{.*}}, {{.*}}, %arg5, {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
amdgpu.wmma 16x16x64 %arg1 * %arg1 + %arg5 : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x64.fp8_bf8 {{.*}}, {{.*}}, %arg4
amdgpu.wmma 16x16x64 %arg1 * %arg2 + %arg4 : vector<32xf8E4M3FN>, vector<32xf8E5M2>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x64.fp8_bf8 {{.*}}, {{.*}}, %arg5 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x64.fp8_bf8 {{.*}}, {{.*}}, %arg5, {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
amdgpu.wmma 16x16x64 %arg1 * %arg2 + %arg5 : vector<32xf8E4M3FN>, vector<32xf8E5M2>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x64.bf8_bf8 {{.*}}, {{.*}}, %arg4
amdgpu.wmma 16x16x64 %arg2 * %arg2 + %arg4 : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x64.bf8_bf8 {{.*}}, {{.*}}, %arg5 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x64.bf8_bf8 {{.*}}, {{.*}}, %arg5, {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
amdgpu.wmma 16x16x64 %arg2 * %arg2 + %arg5 : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x64.bf8_fp8 {{.*}}, {{.*}}, %arg4
amdgpu.wmma 16x16x64 %arg2 * %arg1 + %arg4 : vector<32xf8E5M2>, vector<32xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x64.bf8_fp8 {{.*}}, {{.*}}, %arg5 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x64.bf8_fp8 {{.*}}, {{.*}}, %arg5, {{.*}} : (vector<8xi32>, vector<8xi32>, vector<8xf16>)
amdgpu.wmma 16x16x64 %arg2 * %arg1 + %arg5 : vector<32xf8E5M2>, vector<32xf8E4M3FN>, vector<8xf16>
return
@@ -65,25 +65,25 @@ func.func @wmma_k128(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<64xf8E5M2>,
// CHECK: rocdl.wmma.f32.16x16x128.fp8_fp8 {{.*}}, {{.*}}, %arg2
amdgpu.wmma 16x16x128 %arg0 * %arg0 + %arg2 : vector<64xf8E4M3FN>, vector<64xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x128.fp8_fp8 {{.*}}, {{.*}}, %arg3 {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x128.fp8_fp8 {{.*}}, {{.*}}, %arg3, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
amdgpu.wmma 16x16x128 %arg0 * %arg0 + %arg3 : vector<64xf8E4M3FN>, vector<64xf8E4M3FN>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x128.fp8_bf8 {{.*}}, {{.*}}, %arg2
amdgpu.wmma 16x16x128 %arg0 * %arg1 + %arg2 : vector<64xf8E4M3FN>, vector<64xf8E5M2>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x128.fp8_bf8 {{.*}}, {{.*}}, %arg3 {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x128.fp8_bf8 {{.*}}, {{.*}}, %arg3, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
amdgpu.wmma 16x16x128 %arg0 * %arg1 + %arg3 : vector<64xf8E4M3FN>, vector<64xf8E5M2>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x128.bf8_bf8 {{.*}}, {{.*}}, %arg2
amdgpu.wmma 16x16x128 %arg1 * %arg1 + %arg2 : vector<64xf8E5M2>, vector<64xf8E5M2>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x128.bf8_bf8 {{.*}}, {{.*}}, %arg3 {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x128.bf8_bf8 {{.*}}, {{.*}}, %arg3, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
amdgpu.wmma 16x16x128 %arg1 * %arg1 + %arg3 : vector<64xf8E5M2>, vector<64xf8E5M2>, vector<8xf16>
// CHECK: rocdl.wmma.f32.16x16x128.bf8_fp8 {{.*}}, {{.*}}, %arg2
amdgpu.wmma 16x16x128 %arg1 * %arg0 + %arg2 : vector<64xf8E5M2>, vector<64xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.f16.16x16x128.bf8_fp8 {{.*}}, {{.*}}, %arg3 {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
+ // CHECK: rocdl.wmma.f16.16x16x128.bf8_fp8 {{.*}}, {{.*}}, %arg3, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf16>)
amdgpu.wmma 16x16x128 %arg1 * %arg0 + %arg3 : vector<64xf8E5M2>, vector<64xf8E4M3FN>, vector<8xf16>
return
@@ -92,10 +92,10 @@ func.func @wmma_k128(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<64xf8E5M2>,
// CHECK-LABEL: @wmma_scale_16x16x128_fp8
func.func @wmma_scale_16x16x128_fp8(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<64xf6E2M3FN>,
%arg2 : vector<8xf32>, %arg3 : vector<4xf8E8M0FNU>) {
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp8_e4m3, fmtB = fp8_e4m3, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<16xi32>, vector<16xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%0 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg0) * (%arg3 * %arg0) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf8E4M3FN>, vector<4xf8E8M0FNU>, vector<64xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtA = 2 : i32, fmtB = 2 : i32, scaleAType = 1 : i32} : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp6_e2m3, fmtB = fp6_e2m3, modC = none, scaleAType = row1, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%1 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg1) * (%arg3 * %arg1) + %arg2 {a_first_scale_lane = 16 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf6E2M3FN>, vector<4xf8E8M0FNU>, vector<64xf6E2M3FN>, vector<8xf32>
func.return
@@ -104,10 +104,10 @@ func.func @wmma_scale_16x16x128_fp8(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<
// CHECK-LABEL: @wmma_scale_16x16x128_fp6
func.func @wmma_scale_16x16x128_fp6(%arg0 : vector<64xf6E2M3FN>, %arg1 : vector<64xf6E3M2FN>,
%arg2 : vector<8xf32>, %arg3 : vector<4xf8E8M0FNU>) {
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtA = 2 : i32, fmtB = 2 : i32} : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp6_e2m3, fmtB = fp6_e2m3, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%0 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg0) * (%arg3 * %arg0) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf6E2M3FN>, vector<4xf8E8M0FNU>, vector<64xf6E2M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtA = 3 : i32, fmtB = 3 : i32} : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp6_e3m2, fmtB = fp6_e3m2, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%1 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg1) * (%arg3 * %arg1) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf6E3M2FN>, vector<4xf8E8M0FNU>, vector<64xf6E3M2FN>, vector<8xf32>
func.return
@@ -117,10 +117,10 @@ func.func @wmma_scale_16x16x128_fp6(%arg0 : vector<64xf6E2M3FN>, %arg1 : vector<
func.func @wmma_scale_16x16x128_mixed(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<64xf6E2M3FN>,
%arg2 : vector<64xf4E2M1FN>, %arg3 : vector<8xf32>,
%arg4 : vector<4xf8E8M0FNU>, %arg5 : vector<4xf8E4M3FN>) {
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg3, {{.*}}, {{.*}} {fmtB = 4 : i32, fmtScaleB = 2 : i32} : (vector<16xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg3, {{.*}}, {{.*}}, fmtA = fp8_e4m3, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e4m3 : (vector<16xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%0 = amdgpu.scaled_wmma 16x16x128 (%arg4 * %arg0) * (%arg5 * %arg2) + %arg3 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf8E4M3FN>, vector<4xf8E4M3FN>, vector<64xf4E2M1FN>, vector<8xf32>
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg3, {{.*}}, {{.*}} {fmtA = 2 : i32, fmtB = 4 : i32, fmtScaleB = 2 : i32} : (vector<12xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg3, {{.*}}, {{.*}}, fmtA = fp6_e2m3, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e4m3 : (vector<12xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
%1 = amdgpu.scaled_wmma 16x16x128 (%arg4 * %arg1) * (%arg5 * %arg2) + %arg3 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E8M0FNU>, vector<64xf6E2M3FN>, vector<4xf8E4M3FN>, vector<64xf4E2M1FN>, vector<8xf32>
func.return
@@ -129,10 +129,10 @@ func.func @wmma_scale_16x16x128_mixed(%arg0 : vector<64xf8E4M3FN>, %arg1 : vecto
// CHECK-LABEL: @wmma_scale16_16x16x128_fp8
func.func @wmma_scale16_16x16x128_fp8(%arg0 : vector<64xf8E4M3FN>, %arg1 : vector<64xf6E3M2FN>,
%arg2 : vector<8xf32>, %arg3 : vector<8xf8E8M0FNU>) {
- // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} : (vector<16xi32>, vector<16xi32>, vector<8xf32>, i64, i64) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp8_e4m3, fmtB = fp8_e4m3, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<16xi32>, vector<16xi32>, vector<8xf32>, i64, i64) -> vector<8xf32>
%0 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg0) * (%arg3 * %arg0) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<8xf8E8M0FNU>, vector<64xf8E4M3FN>, vector<8xf8E8M0FNU>, vector<64xf8E4M3FN>, vector<8xf32>
- // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtA = 3 : i32, fmtB = 3 : i32, scaleAType = 1 : i32} : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i64, i64) -> vector<8xf32>
+ // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, fmtA = fp6_e3m2, fmtB = fp6_e3m2, modC = none, scaleAType = row1, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<12xi32>, vector<12xi32>, vector<8xf32>, i64, i64) -> vector<8xf32>
%1 = amdgpu.scaled_wmma 16x16x128 (%arg3 * %arg1) * (%arg3 * %arg1) + %arg2 {a_first_scale_lane = 16 : i32, b_first_scale_lane = 0 : i32} : vector<8xf8E8M0FNU>, vector<64xf6E3M2FN>, vector<8xf8E8M0FNU>, vector<64xf6E3M2FN>, vector<8xf32>
func.return
@@ -141,7 +141,7 @@ func.func @wmma_scale16_16x16x128_fp8(%arg0 : vector<64xf8E4M3FN>, %arg1 : vecto
// CHECK-LABEL: @wmma_scale_32x16x128_fp4
func.func @wmma_scale_32x16x128_fp4(%arg0 : vector<128xf4E2M1FN>, %arg1 : vector<64xf4E2M1FN>,
%arg2 : vector<16xf32>, %arg3 : vector<4xf8E4M3FN>) {
- // CHECK: rocdl.wmma.scale.f32.32x16x128.f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtScaleA = 2 : i32, fmtScaleB = 2 : i32} : (vector<16xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.wmma.scale.f32.32x16x128.f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, modC = none, scaleAType = row0, fmtScaleA = e4m3, scaleBType = row0, fmtScaleB = e4m3 : (vector<16xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
%0 = amdgpu.scaled_wmma 32x16x128 (%arg3 * %arg0) * (%arg3 * %arg1) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<4xf8E4M3FN>, vector<128xf4E2M1FN>, vector<4xf8E4M3FN>, vector<64xf4E2M1FN>, vector<16xf32>
func.return
@@ -150,7 +150,7 @@ func.func @wmma_scale_32x16x128_fp4(%arg0 : vector<128xf4E2M1FN>, %arg1 : vector
// CHECK-LABEL: @wmma_scale16_32x16x128_fp4
func.func @wmma_scale16_32x16x128_fp4(%arg0 : vector<128xf4E2M1FN>, %arg1 : vector<64xf4E2M1FN>,
%arg2 : vector<16xf32>, %arg3 : vector<8xf8E4M3FN>) {
- // CHECK: rocdl.wmma.scale16.f32.32x16x128.f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}} {fmtScaleA = 2 : i32, fmtScaleB = 2 : i32} : (vector<16xi32>, vector<8xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
+ // CHECK: rocdl.wmma.scale16.f32.32x16x128.f4 {{.*}}, {{.*}}, %arg2, {{.*}}, {{.*}}, modC = none, scaleAType = row0, fmtScaleA = e4m3, scaleBType = row0, fmtScaleB = e4m3 : (vector<16xi32>, vector<8xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
%0 = amdgpu.scaled_wmma 32x16x128 (%arg3 * %arg0) * (%arg3 * %arg1) + %arg2 {a_first_scale_lane = 0 : i32, b_first_scale_lane = 0 : i32} : vector<8xf8E4M3FN>, vector<128xf4E2M1FN>, vector<8xf8E4M3FN>, vector<64xf4E2M1FN>, vector<16xf32>
func.return
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index db4f080106a94..3ae1f5df36ed0 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -551,10 +551,12 @@ func.func @lds_barrier() {
// CHECK-LABEL: func @sched_barrier
func.func @sched_barrier() {
- // CHECK: amdgpu.sched_barrier allow = <none>
- amdgpu.sched_barrier allow = <none>
- // CHECK: amdgpu.sched_barrier allow = <valu|all_vmem>
- amdgpu.sched_barrier allow = <valu|all_vmem>
+ // CHECK: amdgpu.sched_barrier allow = none
+ amdgpu.sched_barrier allow = none
+ // CHECK: amdgpu.sched_barrier allow = valu|all_vmem
+ amdgpu.sched_barrier allow = valu|all_vmem
+ // CHECK: amdgpu.sched_barrier allow = all
+ amdgpu.sched_barrier allow = all
func.return
}
diff --git a/mlir/test/Dialect/LLVMIR/rocdl-cache-policy-invalid.mlir b/mlir/test/Dialect/LLVMIR/rocdl-cache-policy-invalid.mlir
new file mode 100644
index 0000000000000..354d1ef661b6e
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/rocdl-cache-policy-invalid.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+llvm.func @pre_gfx12_op_rejects_gfx12(%src : !llvm.ptr<1>,
+ %dst : !llvm.ptr<3>) {
+ // expected-error at +1 {{failed to satisfy constraint: pre-gfx12 or gfx942 AMDGPU cache policy attribute}}
+ rocdl.global.load.async.lds %src, %dst, 4, 0, gfx12<nt> : !llvm.ptr<1>, !llvm.ptr<3>
+ llvm.return
+}
+
+// -----
+
+llvm.func @gfx12_op_rejects_pre_gfx12(%src : !llvm.ptr<1>,
+ %dst : !llvm.ptr<3>) {
+ // expected-error at +1 {{failed to satisfy constraint: gfx12 non-atomic AMDGPU cache policy attribute}}
+ rocdl.global.load.async.to.lds.b32 %src, %dst, 0, pre_gfx12<glc> : !llvm.ptr<1>, !llvm.ptr<3>
+ llvm.return
+}
+
+// -----
+
+llvm.func @non_atomic_buffer_rejects_gfx12_atomic(%rsrc : vector<4xi32>,
+ %offset : i32,
+ %soffset : i32) {
+ // expected-error at +1 {{failed to satisfy constraint: non-atomic AMDGPU buffer cache policy attribute}}
+ %0 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, gfx12_atomic<nt> : i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @atomic_buffer_rejects_gfx12(%rsrc : vector<4xi32>,
+ %offset : i32,
+ %soffset : i32,
+ %vdata : i32) {
+ // expected-error at +1 {{failed to satisfy constraint: atomic AMDGPU buffer cache policy attribute}}
+ %0 = rocdl.raw.buffer.atomic.smax %vdata, %rsrc, %offset, %soffset, gfx12<nt> : i32
+ llvm.return
+}
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index ec63d4c1b7c5c..603913750a32c 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -118,14 +118,22 @@ func.func @rocdl.wave_barrier() {
}
func.func @rocdl.sched_barrier() {
- // CHECK: rocdl.sched.barrier
- rocdl.sched.barrier 0
+ // CHECK: rocdl.sched.barrier none
+ rocdl.sched.barrier none
+ // CHECK: rocdl.sched.barrier valu|all_vmem
+ rocdl.sched.barrier valu|all_vmem
+ // CHECK: rocdl.sched.barrier ldsdma
+ rocdl.sched.barrier ldsdma
+ // CHECK: rocdl.sched.barrier all
+ rocdl.sched.barrier all
llvm.return
}
func.func @rocdl_sched_group_barrier() {
- // CHECK: rocdl.sched.group.barrier
- rocdl.sched.group.barrier 8, 1, 0
+ // CHECK: rocdl.sched.group.barrier mfma_wmma, 1, 0
+ rocdl.sched.group.barrier mfma_wmma, 1, 0
+ // CHECK: rocdl.sched.group.barrier ldsdma, 2, 3
+ rocdl.sched.group.barrier ldsdma, 2, 3
llvm.return
}
@@ -151,153 +159,153 @@ func.func @rocdl.xdlops(%arg0 : f32, %arg1 : f32,
%arg14 : i64, %arg15 : vector<2xf32>,
%arg16: vector<8xbf16>, %arg17 : vector<8xf16>) {
// CHECK-LABEL: rocdl.xdlops
- // CHECK: rocdl.mfma.f32.32x32x1f32 {{.*}} : (f32, f32, vector<32xf32>) -> vector<32xf32>
- %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, 0, 0, 0 :
+ // CHECK: rocdl.mfma.f32.32x32x1f32 {{.*}}, 0, 0, bcast_first_32 : (f32, f32, vector<32xf32>) -> vector<32xf32>
+ %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, 0, 0, bcast_first_32 :
(f32, f32, vector<32xf32>) -> vector<32xf32>
// CHECK: rocdl.mfma.f32.16x16x1f32 {{.*}} : (f32, f32, vector<16xf32>) -> vector<16xf32>
- %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, 0, 0, 0 :
+ %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, 0, 0, none :
(f32, f32, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.4x4x1f32 {{.*}} : (f32, f32, vector<4xf32>) -> vector<4xf32>
- %r2 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, 0, 0, 0 :
+ %r2 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, 0, 0, none :
(f32, f32, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x2f32 {{.*}} : (f32, f32, vector<16xf32>) -> vector<16xf32>
- %r3= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, 0, 0, 0 :
+ %r3= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, 0, 0, none :
(f32, f32, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x4f32 {{.*}} : (f32, f32, vector<4xf32>) -> vector<4xf32>
- %r4 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, 0, 0, 0 :
+ %r4 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, 0, 0, none :
(f32, f32, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<32xf32>) -> vector<32xf32>
- %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, 0, 0, 0 :
+ %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<32xf32>) -> vector<32xf32>
// CHECK: rocdl.mfma.f32.16x16x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<16xf32>) -> vector<16xf32>
- %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, 0, 0, 0 :
+ %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.4x4x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
- %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, 0, 0, 0 :
+ %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x8f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<16xf32>) -> vector<16xf32>
- %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, 0, 0, 0 :
+ %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x16f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
- %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, 0, 0, 0 :
+ %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.i32.32x32x4i8 {{.*}} : (i32, i32, vector<32xi32>) -> vector<32xi32>
- %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, 0, 0, 0 :
+ %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, 0, 0, none :
(i32, i32, vector<32xi32>) -> vector<32xi32>
// CHECK: rocdl.mfma.i32.16x16x4i8 {{.*}} : (i32, i32, vector<16xi32>) -> vector<16xi32>
- %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, 0, 0, 0 :
+ %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, 0, 0, none :
(i32, i32, vector<16xi32>) -> vector<16xi32>
// CHECK: rocdl.mfma.i32.4x4x4i8 {{.*}} : (i32, i32, vector<4xi32>) -> vector<4xi32>
- %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, 0, 0, 0 :
+ %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, 0, 0, none :
(i32, i32, vector<4xi32>) -> vector<4xi32>
// CHECK: rocdl.mfma.i32.32x32x8i8 {{.*}} : (i32, i32, vector<16xi32>) -> vector<16xi32>
- %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, 0, 0, 0 :
+ %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, 0, 0, none :
(i32, i32, vector<16xi32>) -> vector<16xi32>
// CHECK: rocdl.mfma.i32.16x16x16i8 {{.*}} : (i32, i32, vector<4xi32>) -> vector<4xi32>
- %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, 0, 0, 0 :
+ %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, 0, 0, none :
(i32, i32, vector<4xi32>) -> vector<4xi32>
// CHECK: rocdl.mfma.f32.32x32x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<32xf32>) -> vector<32xf32>
- %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, 0, 0, 0 :
+ %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<32xf32>) -> vector<32xf32>
// CHECK: rocdl.mfma.f32.16x16x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<16xf32>) -> vector<16xf32>
- %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, 0, 0, 0 :
+ %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.4x4x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
- %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, 0, 0, 0 :
+ %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x4bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<16xf32>) -> vector<16xf32>
- %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, 0, 0, 0 :
+ %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x8bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
- %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, 0, 0, 0 :
+ %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<32xf32>) -> vector<32xf32>
- %r20 = rocdl.mfma.f32.32x32x4bf16.1k %arg11, %arg11, %arg2, 0, 0, 0 :
+ %r20 = rocdl.mfma.f32.32x32x4bf16.1k %arg11, %arg11, %arg2, 0, 0, none :
(vector<4xi16>, vector<4xi16>, vector<32xf32>) -> vector<32xf32>
// CHECK: rocdl.mfma.f32.16x16x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<16xf32>) -> vector<16xf32>
- %r21 = rocdl.mfma.f32.16x16x4bf16.1k %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r21 = rocdl.mfma.f32.16x16x4bf16.1k %arg11, %arg11, %arg4, 0, 0, none :
(vector<4xi16>, vector<4xi16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.4x4x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<4xf32>) -> vector<4xf32>
- %r22 = rocdl.mfma.f32.4x4x4bf16.1k %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r22 = rocdl.mfma.f32.4x4x4bf16.1k %arg11, %arg11, %arg5, 0, 0, none :
(vector<4xi16>, vector<4xi16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x8bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<16xf32>) -> vector<16xf32>
- %r23 = rocdl.mfma.f32.32x32x8bf16.1k %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r23 = rocdl.mfma.f32.32x32x8bf16.1k %arg11, %arg11, %arg4, 0, 0, none :
(vector<4xi16>, vector<4xi16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x16bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<4xf32>) -> vector<4xf32>
- %r24 = rocdl.mfma.f32.16x16x16bf16.1k %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r24 = rocdl.mfma.f32.16x16x16bf16.1k %arg11, %arg11, %arg5, 0, 0, none :
(vector<4xi16>, vector<4xi16>, vector<4xf32>) -> vector<4xf32>
- // CHECK: rocdl.mfma.f64.16x16x4f64 {{.*}} : (f64, f64, vector<4xf64>) -> vector<4xf64>
- %r25 = rocdl.mfma.f64.16x16x4f64 %arg13, %arg13, %arg12, 0, 0, 0 :
+ // CHECK: rocdl.mfma.f64.16x16x4f64 {{.*}}, 0, 0, neg_a|neg_b : (f64, f64, vector<4xf64>) -> vector<4xf64>
+ %r25 = rocdl.mfma.f64.16x16x4f64 %arg13, %arg13, %arg12, 0, 0, neg_a|neg_b :
(f64, f64, vector<4xf64>) -> vector<4xf64>
// CHECK: rocdl.mfma.f64.4x4x4f64 {{.*}} : (f64, f64, f64) -> f64
- %r26 = rocdl.mfma.f64.4x4x4f64 %arg13, %arg13, %arg13, 0, 0, 0 :
+ %r26 = rocdl.mfma.f64.4x4x4f64 %arg13, %arg13, %arg13, 0, 0, none :
(f64, f64, f64) -> f64
// CHECK: rocdl.mfma.i32.16x16x32.i8 {{.*}} : (i64, i64, vector<4xi32>) -> vector<4xi32>
- %r27 = rocdl.mfma.i32.16x16x32.i8 %arg14, %arg14, %arg9, 0, 0, 0 :
+ %r27 = rocdl.mfma.i32.16x16x32.i8 %arg14, %arg14, %arg9, 0, 0, none :
(i64, i64, vector<4xi32>) -> vector<4xi32>
// CHECK: rocdl.mfma.i32.32x32x16.i8 {{.*}} : (i64, i64, vector<16xi32>) -> vector<16xi32>
- %r28 = rocdl.mfma.i32.32x32x16.i8 %arg14, %arg14, %arg8, 0, 0, 0 :
+ %r28 = rocdl.mfma.i32.32x32x16.i8 %arg14, %arg14, %arg8, 0, 0, none :
(i64, i64, vector<16xi32>) -> vector<16xi32>
// CHECK: rocdl.mfma.f32.16x16x8.xf32 {{.*}} : (vector<2xf32>, vector<2xf32>, vector<4xf32>) -> vector<4xf32>
- %r29 = rocdl.mfma.f32.16x16x8.xf32 %arg15, %arg15, %arg5, 0, 0, 0 :
+ %r29 = rocdl.mfma.f32.16x16x8.xf32 %arg15, %arg15, %arg5, 0, 0, none :
(vector<2xf32>, vector<2xf32>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x4.xf32 {{.*}} : (vector<2xf32>, vector<2xf32>, vector<16xf32>) -> vector<16xf32>
- %r30 = rocdl.mfma.f32.32x32x4.xf32 %arg15, %arg15, %arg4, 0, 0, 0 :
+ %r30 = rocdl.mfma.f32.32x32x4.xf32 %arg15, %arg15, %arg4, 0, 0, none :
(vector<2xf32>, vector<2xf32>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.f32.16x16x32.bf16 {{.*}} : (vector<8xbf16>, vector<8xbf16>, vector<4xf32>) -> vector<4xf32>
- %r31 = rocdl.mfma.f32.16x16x32.bf16 %arg16, %arg16, %arg5, 0, 0, 0 :
+ %r31 = rocdl.mfma.f32.16x16x32.bf16 %arg16, %arg16, %arg5, 0, 0, none :
(vector<8xbf16>, vector<8xbf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.i32.16x16x64.i8 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<4xi32>) -> vector<4xi32>
- %r32 = rocdl.mfma.i32.16x16x64.i8 %arg9, %arg9, %arg9, 0, 0, 0 :
+ %r32 = rocdl.mfma.i32.16x16x64.i8 %arg9, %arg9, %arg9, 0, 0, none :
(vector<4xi32>, vector<4xi32>, vector<4xi32>) -> vector<4xi32>
// CHECK: rocdl.mfma.f32.16x16x32.f16 {{.*}} : (vector<8xf16>, vector<8xf16>, vector<4xf32>) -> vector<4xf32>
- %r33 = rocdl.mfma.f32.16x16x32.f16 %arg17, %arg17, %arg5, 0, 0, 0 :
+ %r33 = rocdl.mfma.f32.16x16x32.f16 %arg17, %arg17, %arg5, 0, 0, none :
(vector<8xf16>, vector<8xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: rocdl.mfma.f32.32x32x16.bf16 {{.*}} : (vector<8xbf16>, vector<8xbf16>, vector<16xf32>) -> vector<16xf32>
- %r34 = rocdl.mfma.f32.32x32x16.bf16 %arg16, %arg16, %arg4, 0, 0, 0 :
+ %r34 = rocdl.mfma.f32.32x32x16.bf16 %arg16, %arg16, %arg4, 0, 0, none :
(vector<8xbf16>, vector<8xbf16>, vector<16xf32>) -> vector<16xf32>
// CHECK: rocdl.mfma.i32.32x32x32.i8 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xi32>) -> vector<16xi32>
- %r35 = rocdl.mfma.i32.32x32x32.i8 %arg9, %arg9, %arg8, 0, 0, 0 :
+ %r35 = rocdl.mfma.i32.32x32x32.i8 %arg9, %arg9, %arg8, 0, 0, none :
(vector<4xi32>, vector<4xi32>, vector<16xi32>) -> vector<16xi32>
// CHECK: rocdl.mfma.f32.32x32x16.f16 {{.*}} : (vector<8xf16>, vector<8xf16>, vector<16xf32>) -> vector<16xf32>
- %r36 = rocdl.mfma.f32.32x32x16.f16 %arg17, %arg17, %arg4, 0, 0, 0 :
+ %r36 = rocdl.mfma.f32.32x32x16.f16 %arg17, %arg17, %arg4, 0, 0, none :
(vector<8xf16>, vector<8xf16>, vector<16xf32>) -> vector<16xf32>
llvm.return
@@ -443,127 +451,127 @@ llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,
// CHECK-LABEL: rocdl.mfma.scale.f32.32x32x64.f8f6f4
// fp8 * fp8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r00 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 0, 0, 0, %arg0, 0, %arg0 :
+ %r00 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * bf8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r01 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 0, 1, 0, %arg0, 0, %arg0 :
+ %r01 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * fp6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r02 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 0, 2, 0, %arg0, 0, %arg0 :
+ %r02 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * bf6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r03 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 0, 3, 0, %arg0, 0, %arg0 :
+ %r03 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * fp4
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r04 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, 0, 4, 0, %arg0, 0, %arg0 :
+ %r04 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, fp8_e4m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r10 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 1, 0, 0, %arg0, 0, %arg0 :
+ %r10 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * bf8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r11 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 1, 1, 0, %arg0, 0, %arg0 :
+ %r11 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r12 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 1, 2, 0, %arg0, 0, %arg0 :
+ %r12 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * bf6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r13 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 1, 3, 0, %arg0, 0, %arg0 :
+ %r13 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp4
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r14 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, 1, 4, 0, %arg0, 0, %arg0 :
+ %r14 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, fp8_e5m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r20 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 2, 0, 0, %arg0, 0, %arg0 :
+ %r20 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * bf8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r21 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 2, 1, 0, %arg0, 0, %arg0 :
+ %r21 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r22 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 2, 2, 0, %arg0, 0, %arg0 :
+ %r22 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * bf6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r23 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 2, 3, 0, %arg0, 0, %arg0 :
+ %r23 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp4
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r24 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, 2, 4, 0, %arg0, 0, %arg0 :
+ %r24 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, fp6_e2m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r30 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 3, 0, 0, %arg0, 0, %arg0 :
+ %r30 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * bf8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r31 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 3, 1, 0, %arg0, 0, %arg0 :
+ %r31 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r32 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 3, 2, 0, %arg0, 0, %arg0 :
+ %r32 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * bf6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r33 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 3, 3, 0, %arg0, 0, %arg0 :
+ %r33 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp4
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r34 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, 3, 4, 0, %arg0, 0, %arg0 :
+ %r34 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, fp6_e3m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r40 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, 4, 0, 0, %arg0, 0, %arg0 :
+ %r40 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * bf8
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r41 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, 4, 1, 0, %arg0, 0, %arg0 :
+ %r41 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r42 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, 4, 2, 0, %arg0, 0, %arg0 :
+ %r42 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * bf6
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r43 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, 4, 3, 0, %arg0, 0, %arg0 :
+ %r43 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp4
// CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r44 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg4, %arg1, 4, 4, 0, %arg0, 0, %arg0 :
+ %r44 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg4, %arg1, fp4_e2m1, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
llvm.return
@@ -576,127 +584,127 @@ llvm.func @rocdl.mfma.scale.f32.16x16x128.f8f6f4(%arg0 : i32,
// CHECK-LABEL: rocdl.mfma.scale.f32.16x16x128.f8f6f4
// fp8 * fp8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r00 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 0, 0, 0, %arg0, 0, %arg0 :
+ %r00 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * bf8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r01 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 0, 1, 0, %arg0, 0, %arg0 :
+ %r01 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * fp6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r02 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 0, 2, 0, %arg0, 0, %arg0 :
+ %r02 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * bf6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r03 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 0, 3, 0, %arg0, 0, %arg0 :
+ %r03 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * fp4
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r04 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, 0, 4, 0, %arg0, 0, %arg0 :
+ %r04 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, fp8_e4m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r10 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 1, 0, 0, %arg0, 0, %arg0 :
+ %r10 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * bf8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r11 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 1, 1, 0, %arg0, 0, %arg0 :
+ %r11 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r12 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 1, 2, 0, %arg0, 0, %arg0 :
+ %r12 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * bf6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r13 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 1, 3, 0, %arg0, 0, %arg0 :
+ %r13 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp4
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r14 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, 1, 4, 0, %arg0, 0, %arg0 :
+ %r14 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, fp8_e5m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r20 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 2, 0, 0, %arg0, 0, %arg0 :
+ %r20 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * bf8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r21 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 2, 1, 0, %arg0, 0, %arg0 :
+ %r21 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r22 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 2, 2, 0, %arg0, 0, %arg0 :
+ %r22 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * bf6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r23 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 2, 3, 0, %arg0, 0, %arg0 :
+ %r23 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp4
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r24 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, 2, 4, 0, %arg0, 0, %arg0 :
+ %r24 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, fp6_e2m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r30 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 3, 0, 0, %arg0, 0, %arg0 :
+ %r30 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * bf8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r31 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 3, 1, 0, %arg0, 0, %arg0 :
+ %r31 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r32 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 3, 2, 0, %arg0, 0, %arg0 :
+ %r32 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * bf6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r33 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 3, 3, 0, %arg0, 0, %arg0 :
+ %r33 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp4
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r34 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, 3, 4, 0, %arg0, 0, %arg0 :
+ %r34 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, fp6_e3m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r40 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, 4, 0, 0, %arg0, 0, %arg0 :
+ %r40 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * bf8
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r41 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, 4, 1, 0, %arg0, 0, %arg0 :
+ %r41 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r42 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, 4, 2, 0, %arg0, 0, %arg0 :
+ %r42 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * bf6
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r43 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, 4, 3, 0, %arg0, 0, %arg0 :
+ %r43 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp4
// CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r44 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg4, %arg1, 4, 4, 0, %arg0, 0, %arg0 :
+ %r44 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg4, %arg1, fp4_e2m1, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
llvm.return
@@ -780,10 +788,10 @@ llvm.func @rocdl.global.load.async.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3>)
llvm.func @rocdl.global.load.async.to.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3>) {
// CHECK-LABEL @rocdl.global.load.async.to.lds
- // CHECK: rocdl.global.load.async.to.lds.b8 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.load.async.to.lds.b32 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.load.async.to.lds.b64 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.load.async.to.lds.b128 %{{.*}}, %{{.*}}, 0, 0
+ // CHECK: rocdl.global.load.async.to.lds.b8 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.load.async.to.lds.b32 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.load.async.to.lds.b64 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.load.async.to.lds.b128 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.load.async.to.lds.b8 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.load.async.to.lds.b32 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.load.async.to.lds.b64 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
@@ -793,10 +801,10 @@ llvm.func @rocdl.global.load.async.to.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3
// CHECK-LABEL: @rocdl.global.store.async.from.lds
llvm.func @rocdl.global.store.async.from.lds(%dst : !llvm.ptr<1>, %src: !llvm.ptr<3>) {
- // CHECK: rocdl.global.store.async.from.lds.b8 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.store.async.from.lds.b32 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.store.async.from.lds.b64 %{{.*}}, %{{.*}}, 0, 0
- // CHECK: rocdl.global.store.async.from.lds.b128 %{{.*}}, %{{.*}}, 0, 0
+ // CHECK: rocdl.global.store.async.from.lds.b8 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.store.async.from.lds.b32 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.store.async.from.lds.b64 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
+ // CHECK: rocdl.global.store.async.from.lds.b128 %{{.*}}, %{{.*}}, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.store.async.from.lds.b8 %dst, %src, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.store.async.from.lds.b32 %dst, %src, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
rocdl.global.store.async.from.lds.b64 %dst, %src, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>
@@ -822,8 +830,8 @@ llvm.func @rocdl.cluster.load.async.to.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<
llvm.func @rocdl.tensor.load.to.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>,
%dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>,
%dgroup4 : vector<8xi32>) {
- // CHECK: rocdl.tensor.load.to.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>
- rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ // CHECK: rocdl.tensor.load.to.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4, 0 : vector<4xi32>, vector<8xi32>
llvm.return
}
@@ -831,8 +839,8 @@ llvm.func @rocdl.tensor.load.to.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<
llvm.func @rocdl.tensor.store.from.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>,
%dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>,
%dgroup4 : vector<8xi32>) {
- // CHECK: rocdl.tensor.store.from.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>
- rocdl.tensor.store.from.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ // CHECK: rocdl.tensor.store.from.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.store.from.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4, 0 : vector<4xi32>, vector<8xi32>
llvm.return
}
@@ -851,27 +859,27 @@ llvm.func @rocdl.raw.ptr.buffer.f32(%rsrc : !llvm.ptr<8>,
%aux : i32, %vdata1 : f32,
%vdata2 : vector<2xf32>, %vdata4 : vector<4xf32>) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : vector<2xf32>
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : vector<4xf32>
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<2xf32>
- // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<4xf32>
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
+ // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
- %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : f32
- %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xf32>
- %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xf32>
+ %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : f32
+ %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xf32>
+ %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xf32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : f32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xf32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, %aux : vector<4xf32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xf32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, 0 : vector<4xf32>
- %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32
- %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32
+ %atomic_fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %atomic_fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
llvm.return
}
@@ -880,8 +888,8 @@ llvm.func @rocdl.raw.ptr.buffer.load.lds(%rsrc : !llvm.ptr<8>, %dstLds : !llvm.p
%size: i32, %voffset : i32, %soffset : i32, %offset : i32,
%aux : i32) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.load.lds
- // CHECK: rocdl.raw.ptr.buffer.load.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}
- rocdl.raw.ptr.buffer.load.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, %aux
+ // CHECK: rocdl.raw.ptr.buffer.load.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0
+ rocdl.raw.ptr.buffer.load.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, 0
llvm.return
}
@@ -890,8 +898,8 @@ llvm.func @rocdl.raw.ptr.buffer.load.async.lds(%rsrc : !llvm.ptr<8>, %dstLds : !
%size: i32, %voffset : i32, %soffset : i32, %offset : i32,
%aux : i32) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.load.async.lds
- // CHECK: rocdl.raw.ptr.buffer.load.async.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}
- rocdl.raw.ptr.buffer.load.async.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, %aux
+ // CHECK: rocdl.raw.ptr.buffer.load.async.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0
+ rocdl.raw.ptr.buffer.load.async.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, 0
llvm.return
}
@@ -901,27 +909,46 @@ llvm.func @rocdl.raw.ptr.buffer.i32(%rsrc : !llvm.ptr<8>,
%aux : i32, %vdata1 : i32,
%vdata2 : vector<2xi32>, %vdata4 : vector<4xi32>) {
// CHECK-LABEL: rocdl.raw.ptr.buffer.i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32
- // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, %aux : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, 0 : i32
llvm.return
}
llvm.func @rocdl.global.prefetch(%ptr : !llvm.ptr<1>) {
// CHECK-LABEL: rocdl.global.prefetch
- // CHECK: rocdl.global.prefetch %{{.*}}, scope 0 : !llvm.ptr<1>
- rocdl.global.prefetch %ptr, scope 0 : !llvm.ptr<1>
+ // CHECK: rocdl.global.prefetch %{{.*}}, 0 : !llvm.ptr<1>
+ rocdl.global.prefetch %ptr, 0 : !llvm.ptr<1>
llvm.return
}
llvm.func @rocdl.flat.prefetch(%ptr : !llvm.ptr) {
// CHECK-LABEL: rocdl.flat.prefetch
- // CHECK: rocdl.flat.prefetch %{{.*}}, scope 0 : !llvm.ptr
- rocdl.flat.prefetch %ptr, scope 0 : !llvm.ptr
+ // CHECK: rocdl.flat.prefetch %{{.*}}, 0 : !llvm.ptr
+ rocdl.flat.prefetch %ptr, 0 : !llvm.ptr
+ llvm.return
+}
+
+llvm.func @rocdl.cachepolicy.enums(%rsrc : vector<4xi32>, %offset : i32,
+ %soffset : i32, %vdata : i32,
+ %ptr : !llvm.ptr<1>) {
+ // CHECK-LABEL: rocdl.cachepolicy.enums
+ // CHECK: rocdl.raw.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, pre_gfx12<glc|slc> : i32
+ // CHECK: rocdl.raw.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, gfx942<sc0|nt> : i32
+ // CHECK: rocdl.global.prefetch %{{.*}}, gfx12<nt_rt|scope_se> : !llvm.ptr<1>
+ // CHECK: rocdl.global.prefetch %{{.*}}, gfx12<scope_sys|scal> : !llvm.ptr<1>
+ // CHECK: rocdl.raw.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, gfx12_atomic<nv> : i32
+ // CHECK: rocdl.raw.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 2 : i32
+ %r = rocdl.raw.buffer.load %rsrc, %offset, %soffset, pre_gfx12<glc|slc> : i32
+ %r942 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, gfx942<sc0|nt> : i32
+ rocdl.global.prefetch %ptr, gfx12<nt_rt|scope_se> : !llvm.ptr<1>
+ rocdl.global.prefetch %ptr, gfx12<scope_sys|scal> : !llvm.ptr<1>
+ rocdl.raw.buffer.atomic.smax %vdata, %rsrc, %offset, %soffset, gfx12_atomic<nv> : i32
+ rocdl.raw.buffer.atomic.smax %vdata, %rsrc, %offset, %soffset, 2 : i32
llvm.return
}
@@ -941,27 +968,27 @@ llvm.func @rocdl.raw.buffer.f32(%rsrc : vector<4xi32>,
%aux : i32, %vdata1 : f32,
%vdata2 : vector<2xf32>, %vdata4 : vector<4xf32>) {
// CHECK-LABEL: rocdl.raw.buffer.f32
- // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32
- // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<2xf32>
- // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<4xf32>
+ // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
+ // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
- // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32
- // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<2xf32>
- // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<4xf32>
+ // CHECK: rocdl.raw.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: rocdl.raw.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<2xf32>
+ // CHECK: rocdl.raw.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : vector<4xf32>
- // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.fadd %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32
- // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.fmax %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32
+ // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
+ // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : f32
- %r1 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : f32
- %r2 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xf32>
- %r4 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xf32>
+ %r1 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : f32
+ %r2 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xf32>
+ %r4 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xf32>
- rocdl.raw.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : f32
- rocdl.raw.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xf32>
- rocdl.raw.buffer.store %vdata4, %rsrc, %offset, %offset, %aux : vector<4xf32>
+ rocdl.raw.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ rocdl.raw.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xf32>
+ rocdl.raw.buffer.store %vdata4, %rsrc, %offset, %offset, 0 : vector<4xf32>
- %atomic_fadd = rocdl.raw.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32
- %atomic_fmax = rocdl.raw.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32
+ %atomic_fadd = rocdl.raw.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %atomic_fmax = rocdl.raw.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
llvm.return
}
@@ -972,13 +999,13 @@ llvm.func @rocdl.raw.buffer.i32(%rsrc : vector<4xi32>,
%aux : i32, %vdata1 : i32,
%vdata2 : vector<2xi32>, %vdata4 : vector<4xi32>) {
// CHECK-LABEL: rocdl.raw.buffer.i32
- // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.smax %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32
- // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.umin %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32
- // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.cmpswap(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : i32, vector<4xi32>
+ // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, 0 : i32
+ // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.cmpswap(%{{.*}}, %{{.*}}, 0) : i32, vector<4xi32>
- %smax = rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %umin = rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %val = rocdl.raw.buffer.atomic.cmpswap(%vdata1, %vdata1, %rsrc, %offset, %soffset, %aux) : i32, vector<4xi32>
+ %smax = rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %umin = rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %val = rocdl.raw.buffer.atomic.cmpswap(%vdata1, %vdata1, %rsrc, %offset, %soffset, 0) : i32, vector<4xi32>
llvm.return
}
@@ -1526,17 +1553,17 @@ llvm.func @rocdl.cvt.scalef32.sr.pk16(%v16xf32: vector<16xf32>,
// CHECK-LABEL: @rocdl_wmma_scale_ops
llvm.func @rocdl_wmma_scale_ops(%a_f8: vector<8xi32>, %a_f4: vector<4xi32>, %c_f32: vector<4xf32>, %c16_f32: vector<16xf32>,
%scale_i32: i32, %scale_i64: i64) {
- // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- %r0 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i32, %scale_i32 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, fmtA = fp4_e2m1, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
+ %r0 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i32, %scale_i32, fmtA = fp4_e2m1, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
- // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>
- %r1 = rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i64, %scale_i64 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>
+ // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, fmtA = fp4_e2m1, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>
+ %r1 = rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i64, %scale_i64, fmtA = fp4_e2m1, fmtB = fp4_e2m1, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>
- // CHECK: rocdl.wmma.scale.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- %r2 = rocdl.wmma.scale.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i32, %scale_i32 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ // CHECK: rocdl.wmma.scale.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
+ %r2 = rocdl.wmma.scale.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i32, %scale_i32, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
- // CHECK: rocdl.wmma.scale16.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
- %r3 = rocdl.wmma.scale16.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i64, %scale_i64 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
+ // CHECK: rocdl.wmma.scale16.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
+ %r3 = rocdl.wmma.scale16.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i64, %scale_i64, modC = none, scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>
llvm.return
}
diff --git a/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir b/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
index e4d2ad48644ae..ee289b9cd5549 100644
--- a/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
+++ b/mlir/test/Integration/GPU/ROCM/gpu-lower-to-rocdl-pipeline.mlir
@@ -23,7 +23,7 @@ func.func @vecadd(%arg0 : memref<5xf32>, %arg1 : memref<5xf32>, %arg2 : memref<5
threads(%tx, %ty, %tz) in (%block_x = %block_dim, %block_y = %c1, %block_z = %c1) {
%a = memref.load %arg0[%tx] : memref<5xf32>
%b = memref.load %arg1[%tx] : memref<5xf32>
- amdgpu.sched_barrier allow = <none>
+ amdgpu.sched_barrier allow = none
%c = arith.addf %a, %b : f32
memref.store %c, %arg2[%tx] : memref<5xf32>
gpu.terminator
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 0a3127b868a3d..f3233f3e50bce 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -405,16 +405,24 @@ llvm.func @rocdl.setprio() {
llvm.func @rocdl.schedbarrier() {
// CHECK: call void @llvm.amdgcn.sched.barrier(i32 0)
- rocdl.sched.barrier 0
+ rocdl.sched.barrier none
// CHECK-NEXT: call void @llvm.amdgcn.sched.barrier(i32 1)
- rocdl.sched.barrier 1
+ rocdl.sched.barrier non_mem_non_sideeffect
+ // CHECK-NEXT: call void @llvm.amdgcn.sched.barrier(i32 18)
+ rocdl.sched.barrier valu|all_vmem
+ // CHECK-NEXT: call void @llvm.amdgcn.sched.barrier(i32 2048)
+ rocdl.sched.barrier ldsdma
+ // CHECK-NEXT: call void @llvm.amdgcn.sched.barrier(i32 4095)
+ rocdl.sched.barrier all
llvm.return
}
llvm.func @rocdl.sched.group.barrier() {
// CHECK-LABEL: rocdl.sched.group.barrier
// CHECK-NEXT: call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
- rocdl.sched.group.barrier 8, 1, 0
+ rocdl.sched.group.barrier mfma_wmma, 1, 0
+ // CHECK-NEXT: call void @llvm.amdgcn.sched.group.barrier(i32 2048, i32 2, i32 3)
+ rocdl.sched.group.barrier ldsdma, 2, 3
llvm.return
}
@@ -434,145 +442,150 @@ llvm.func @rocdl.xdlops(%arg0 : f32, %arg1 : f32,
%arg8 : vector<16 x i32>, %arg9 : vector<4xi32>,
%arg10 : vector<2xi16>, %arg11 : i64,
%arg12 : vector<8xbf16>, %arg13 : vector<4xi32>,
- %arg14 : vector<8xf16>) -> vector<32 x f32> {
+ %arg14 : vector<8xf16>, %arg15 : f64,
+ %arg16 : vector<4xf64>) -> vector<32 x f32> {
// CHECK-LABEL: rocdl.xdlops
- // CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %{{.*}}, float %{{.*}}, <32 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, 0, 0, 0 :
+ // CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %{{.*}}, float %{{.*}}, <32 x float> %{{.*}}, i32 0, i32 0, i32 1)
+ %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, 0, 0, bcast_first_32 :
(f32, f32, vector<32 x f32>) -> vector<32 x f32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float %{{.*}}, float %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, 0, 0, 0 :
+ %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, 0, 0, none :
(f32, f32, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x4f32(float %{{.*}}, float %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r2 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, 0, 0, 0 :
+ %r2 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, 0, 0, none :
(f32, f32, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float %{{.*}}, float %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r3 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, 0, 0, 0 :
+ %r3 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, 0, 0, none :
(f32, f32, vector<4xf32>) -> vector<4xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %{{.*}}, float %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r4= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, 0, 0, 0 :
+ %r4= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, 0, 0, none :
(f32, f32, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <32 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, 0, 0, 0 :
+ %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<32 x f32>) -> vector<32 x f32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, 0, 0, 0 :
+ %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, 0, 0, 0 :
+ %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, 0, 0, 0 :
+ %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, 0, 0, 0 :
+ %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, 0, 0, none :
(vector<4xf16>, vector<4xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 %{{.*}}, i32 %{{.*}}, <32 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, 0, 0, 0 :
+ %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, 0, 0, none :
(i32, i32, vector<32 x i32>) -> vector<32 x i32>
// CHECK: call <16 x i32> @llvm.amdgcn.mfma.i32.16x16x4i8(i32 %{{.*}}, i32 %{{.*}}, <16 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, 0, 0, 0 :
+ %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, 0, 0, none :
(i32, i32, vector<16 x i32>) -> vector<16 x i32>
// CHECK: call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, 0, 0, 0 :
+ %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, 0, 0, none :
(i32, i32, vector<4xi32>) -> vector<4xi32>
// CHECK: call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 %{{.*}}, i32 %{{.*}}, <16 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, 0, 0, 0 :
+ %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, 0, 0, none :
(i32, i32, vector<16 x i32>) -> vector<16 x i32>
// CHECK: call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, 0, 0, 0 :
+ %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, 0, 0, none :
(i32, i32, vector<4xi32>) -> vector<4xi32>
// CHECK: call <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <32 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, 0, 0, 0 :
+ %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<32 x f32>) -> vector<32 x f32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, 0, 0, 0 :
+ %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.4x4x2bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, 0, 0, 0 :
+ %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, 0, 0, 0 :
+ %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<16 x f32>) -> vector<16 x f32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, 0, 0, 0 :
+ %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, 0, 0, none :
(vector<2xi16>, vector<2xi16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8(i64 %{{.*}}, i64 %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r20 = rocdl.mfma.f32.16x16x32.bf8.bf8 %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r20 = rocdl.mfma.f32.16x16x32.bf8.bf8 %arg11, %arg11, %arg5, 0, 0, none :
(i64, i64, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8(i64 %{{.*}}, i64 %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r21 = rocdl.mfma.f32.16x16x32.bf8.fp8 %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r21 = rocdl.mfma.f32.16x16x32.bf8.fp8 %arg11, %arg11, %arg5, 0, 0, none :
(i64, i64, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8(i64 %{{.*}}, i64 %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r22 = rocdl.mfma.f32.16x16x32.fp8.bf8 %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r22 = rocdl.mfma.f32.16x16x32.fp8.bf8 %arg11, %arg11, %arg5, 0, 0, none :
(i64, i64, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %{{.*}}, i64 %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r23 = rocdl.mfma.f32.16x16x32.fp8.fp8 %arg11, %arg11, %arg5, 0, 0, 0 :
+ %r23 = rocdl.mfma.f32.16x16x32.fp8.fp8 %arg11, %arg11, %arg5, 0, 0, none :
(i64, i64, vector<4xf32>) -> vector<4xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64 %{{.*}}, i64 %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r24 = rocdl.mfma.f32.32x32x16.bf8.bf8 %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r24 = rocdl.mfma.f32.32x32x16.bf8.bf8 %arg11, %arg11, %arg4, 0, 0, none :
(i64, i64, vector<16xf32>) -> vector<16xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8(i64 %{{.*}}, i64 %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r25 = rocdl.mfma.f32.32x32x16.bf8.fp8 %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r25 = rocdl.mfma.f32.32x32x16.bf8.fp8 %arg11, %arg11, %arg4, 0, 0, none :
(i64, i64, vector<16xf32>) -> vector<16xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8(i64 %{{.*}}, i64 %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r26 = rocdl.mfma.f32.32x32x16.fp8.bf8 %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r26 = rocdl.mfma.f32.32x32x16.fp8.bf8 %arg11, %arg11, %arg4, 0, 0, none :
(i64, i64, vector<16xf32>) -> vector<16xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64 %{{.*}}, i64 %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r27 = rocdl.mfma.f32.32x32x16.bf8.bf8 %arg11, %arg11, %arg4, 0, 0, 0 :
+ %r27 = rocdl.mfma.f32.32x32x16.bf8.bf8 %arg11, %arg11, %arg4, 0, 0, none :
(i64, i64, vector<16xf32>) -> vector<16xf32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r28 = rocdl.mfma.f32.16x16x32.bf16 %arg12, %arg12, %arg5, 0, 0, 0 :
+ %r28 = rocdl.mfma.f32.16x16x32.bf16 %arg12, %arg12, %arg5, 0, 0, none :
(vector<8xbf16>, vector<8xbf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r29 = rocdl.mfma.i32.16x16x64.i8 %arg9, %arg9, %arg9, 0, 0, 0 :
+ %r29 = rocdl.mfma.i32.16x16x64.i8 %arg9, %arg9, %arg9, 0, 0, none :
(vector<4xi32>, vector<4xi32>, vector<4xi32>) -> vector<4xi32>
// CHECK: call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <4 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r30 = rocdl.mfma.f32.16x16x32.f16 %arg14, %arg14, %arg5, 0, 0, 0 :
+ %r30 = rocdl.mfma.f32.16x16x32.f16 %arg14, %arg14, %arg5, 0, 0, none :
(vector<8xf16>, vector<8xf16>, vector<4xf32>) -> vector<4xf32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %1{{.*}}, <8 x bfloat> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r31 = rocdl.mfma.f32.32x32x16.bf16 %arg12, %arg12, %arg4, 0, 0, 0 :
+ %r31 = rocdl.mfma.f32.32x32x16.bf16 %arg12, %arg12, %arg4, 0, 0, none :
(vector<8xbf16>, vector<8xbf16>, vector<16xf32>) -> vector<16xf32>
// CHECK: call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r32 = rocdl.mfma.i32.32x32x32.i8 %arg9, %arg9, %arg8, 0, 0, 0 :
+ %r32 = rocdl.mfma.i32.32x32x32.i8 %arg9, %arg9, %arg8, 0, 0, none :
(vector<4xi32>, vector<4xi32>, vector<16xi32>) -> vector<16xi32>
// CHECK: call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <16 x float> %{{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}})
- %r33 = rocdl.mfma.f32.32x32x16.f16 %arg14, %arg14, %arg4, 0, 0, 0 :
+ %r33 = rocdl.mfma.f32.32x32x16.f16 %arg14, %arg14, %arg4, 0, 0, none :
(vector<8xf16>, vector<8xf16>, vector<16xf32>) -> vector<16xf32>
+ // CHECK: call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %{{.*}}, double %{{.*}}, <4 x double> %{{.*}}, i32 0, i32 0, i32 3)
+ %r34 = rocdl.mfma.f64.16x16x4f64 %arg15, %arg15, %arg16, 0, 0, neg_a|neg_b :
+ (f64, f64, vector<4xf64>) -> vector<4xf64>
+
llvm.return %r0 : vector<32 x f32>
}
@@ -717,127 +730,127 @@ llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,
// CHECK-LABEL: rocdl.mfma.scale.f32.32x32x64.f8f6f4
// fp8 * fp8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r00 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 0, 0, 0, %arg0, 0, %arg0 :
+ %r00 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * bf8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r01 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 0, 1, 0, %arg0, 0, %arg0 :
+ %r01 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * fp6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r02 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 0, 2, 0, %arg0, 0, %arg0 :
+ %r02 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * bf6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r03 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 0, 3, 0, %arg0, 0, %arg0 :
+ %r03 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp8 * fp4
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v4i32(<8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 0, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r04 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, 0, 4, 0, %arg0, 0, %arg0 :
+ %r04 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, fp8_e4m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 1, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r10 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 1, 0, 0, %arg0, 0, %arg0 :
+ %r10 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * bf8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 1, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r11 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, 1, 1, 0, %arg0, 0, %arg0 :
+ %r11 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 1, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r12 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 1, 2, 0, %arg0, 0, %arg0 :
+ %r12 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * bf6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 1, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r13 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, 1, 3, 0, %arg0, 0, %arg0 :
+ %r13 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf8 * fp4
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v4i32(<8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 1, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r14 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, 1, 4, 0, %arg0, 0, %arg0 :
+ %r14 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, fp8_e5m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 2, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r20 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 2, 0, 0, %arg0, 0, %arg0 :
+ %r20 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * bf8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 2, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r21 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 2, 1, 0, %arg0, 0, %arg0 :
+ %r21 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 2, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r22 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 2, 2, 0, %arg0, 0, %arg0 :
+ %r22 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * bf6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 2, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r23 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 2, 3, 0, %arg0, 0, %arg0 :
+ %r23 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp6 * fp4
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v4i32(<6 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 2, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r24 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, 2, 4, 0, %arg0, 0, %arg0 :
+ %r24 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, fp6_e2m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 3, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r30 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 3, 0, 0, %arg0, 0, %arg0 :
+ %r30 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * bf8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 3, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r31 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, 3, 1, 0, %arg0, 0, %arg0 :
+ %r31 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 3, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r32 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 3, 2, 0, %arg0, 0, %arg0 :
+ %r32 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * bf6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 3, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r33 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, 3, 3, 0, %arg0, 0, %arg0 :
+ %r33 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// bf6 * fp4
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v6i32.v4i32(<6 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 3, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r34 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, 3, 4, 0, %arg0, 0, %arg0 :
+ %r34 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, fp6_e3m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v4i32.v8i32(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 4, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r40 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, 4, 0, 0, %arg0, 0, %arg0 :
+ %r40 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * bf8
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v4i32.v8i32(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 4, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r41 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, 4, 1, 0, %arg0, 0, %arg0 :
+ %r41 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v4i32.v6i32(<4 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 4, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r42 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, 4, 2, 0, %arg0, 0, %arg0 :
+ %r42 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * bf6
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v4i32.v6i32(<4 x i32> %{{.*}}, <6 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 4, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r43 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, 4, 3, 0, %arg0, 0, %arg0 :
+ %r43 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
// fp4 * fp4
// CHECK: call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v4i32.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x float> %{{.*}}, i32 4, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r44 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg4, %arg1, 4, 4, 0, %arg0, 0, %arg0 :
+ %r44 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg4, %arg1, fp4_e2m1, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>
llvm.return %r00 : vector<16 x f32>
@@ -850,127 +863,127 @@ llvm.func @rocdl.mfma.scale.f32.16x16x128.f8f6f4(%arg0 : i32,
// CHECK-LABEL: rocdl.mfma.scale.f32.16x16x128.f8f6f4
// fp8 * fp8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r00 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 0, 0, 0, %arg0, 0, %arg0 :
+ %r00 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * bf8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r01 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 0, 1, 0, %arg0, 0, %arg0 :
+ %r01 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e4m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * fp6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r02 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 0, 2, 0, %arg0, 0, %arg0 :
+ %r02 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * bf6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r03 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 0, 3, 0, %arg0, 0, %arg0 :
+ %r03 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e4m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp8 * fp4
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 0, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r04 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, 0, 4, 0, %arg0, 0, %arg0 :
+ %r04 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, fp8_e4m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 1, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r10 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 1, 0, 0, %arg0, 0, %arg0 :
+ %r10 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * bf8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 1, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r11 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, 1, 1, 0, %arg0, 0, %arg0 :
+ %r11 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, fp8_e5m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 1, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r12 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 1, 2, 0, %arg0, 0, %arg0 :
+ %r12 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * bf6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v6i32(<8 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 1, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r13 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, 1, 3, 0, %arg0, 0, %arg0 :
+ %r13 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, fp8_e5m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf8 * fp4
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v4i32(<8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 1, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r14 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, 1, 4, 0, %arg0, 0, %arg0 :
+ %r14 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, fp8_e5m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 2, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r20 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 2, 0, 0, %arg0, 0, %arg0 :
+ %r20 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * bf8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 2, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r21 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 2, 1, 0, %arg0, 0, %arg0 :
+ %r21 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e2m3, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 2, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r22 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 2, 2, 0, %arg0, 0, %arg0 :
+ %r22 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * bf6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 2, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r23 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 2, 3, 0, %arg0, 0, %arg0 :
+ %r23 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e2m3, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp6 * fp4
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 2, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r24 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, 2, 4, 0, %arg0, 0, %arg0 :
+ %r24 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, fp6_e2m3, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 3, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r30 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 3, 0, 0, %arg0, 0, %arg0 :
+ %r30 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * bf8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v8i32(<6 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 3, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r31 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, 3, 1, 0, %arg0, 0, %arg0 :
+ %r31 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, fp6_e3m2, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 3, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r32 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 3, 2, 0, %arg0, 0, %arg0 :
+ %r32 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * bf6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 3, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r33 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, 3, 3, 0, %arg0, 0, %arg0 :
+ %r33 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, fp6_e3m2, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// bf6 * fp4
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v4i32(<6 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 3, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r34 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, 3, 4, 0, %arg0, 0, %arg0 :
+ %r34 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, fp6_e3m2, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 4, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r40 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, 4, 0, 0, %arg0, 0, %arg0 :
+ %r40 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e4m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * bf8
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v8i32(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 4, i32 1, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r41 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, 4, 1, 0, %arg0, 0, %arg0 :
+ %r41 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, fp4_e2m1, fp8_e5m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 4, i32 2, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r42 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, 4, 2, 0, %arg0, 0, %arg0 :
+ %r42 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e2m3, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * bf6
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v6i32(<4 x i32> %{{.*}}, <6 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 4, i32 3, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}})
- %r43 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, 4, 3, 0, %arg0, 0, %arg0 :
+ %r43 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, fp4_e2m1, fp6_e3m2, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// fp4 * fp4
// CHECK: call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x float> %{{.*}}, i32 4, i32 4, i32 0, i32 %{{.*}}, i32 0, i32 %{{.*}}
- %r44 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg4, %arg1, 4, 4, 0, %arg0, 0, %arg0 :
+ %r44 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg4, %arg1, fp4_e2m1, fp4_e2m1, 0, %arg0, 0, %arg0 :
(vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
llvm.return %r00 : vector<4 x f32>
@@ -1036,76 +1049,76 @@ llvm.func @rocdl.wmma(%arg0 : vector<8xf32>, %arg1 : vector<16 x f16>, %arg2 : v
// f32 -> f32
// CHECK: call <4 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v4f32.v16f32(<16 x float> %{{.*}}, <16 x float> %{{.*}} i16 0, <4 x float> %{{.*}} i1 false, i1 false)
- %r1.gfx1250 = rocdl.wmma.f32.16x16x4.f32 %arg10, %arg10, %arg11 {signA = false, signB = false, modC = 0 : i16} : (vector<16xf32>, vector<16xf32>, vector<4xf32>) -> vector<4xf32>
+ %r1.gfx1250 = rocdl.wmma.f32.16x16x4.f32 %arg10, %arg10, %arg11, modC = none : (vector<16xf32>, vector<16xf32>, vector<4xf32>) -> vector<4xf32>
// f16 -> f32
// CHECK: call <32 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v32f32.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}} i16 0, <32 x float> %{{.*}} i1 false, i1 false)
- %r2.gfx1250 = rocdl.wmma.f32.16x16x32.f16 %arg1, %arg1, %arg12 {signA = false, signB = false, modC = 0 : i16} : (vector<16xf16>, vector<16xf16>, vector<32xf32>) -> vector<32xf32>
+ %r2.gfx1250 = rocdl.wmma.f32.16x16x32.f16 %arg1, %arg1, %arg12, modC = none : (vector<16xf16>, vector<16xf16>, vector<32xf32>) -> vector<32xf32>
// bf16 -> f32
// CHECK: call <32 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v32f32.v16bf16(<16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}} i16 0, <32 x float> %{{.*}} i1 false, i1 false)
- %r3.gfx1250 = rocdl.wmma.f32.16x16x32.bf16 %arg16, %arg16, %arg12 {signA = false, signB = false, modC = 0 : i16} : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xf32>
+ %r3.gfx1250 = rocdl.wmma.f32.16x16x32.bf16 %arg16, %arg16, %arg12, modC = none : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xf32>
// f16 -> f16
// CHECK: call <32 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v32f16.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}} i16 0, <32 x half> %{{.*}} i1 false, i1 false)
- %r4.gfx1250 = rocdl.wmma.f16.16x16x32.f16 %arg1, %arg1, %arg9 {signA = false, signB = false, modC = 0 : i16} : (vector<16xf16>, vector<16xf16>, vector<32xf16>) -> vector<32xf16>
+ %r4.gfx1250 = rocdl.wmma.f16.16x16x32.f16 %arg1, %arg1, %arg9, modC = none : (vector<16xf16>, vector<16xf16>, vector<32xf16>) -> vector<32xf16>
// bf16 -> bf16
// CHECK: call <32 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v32bf16.v16bf16(<16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}} i16 0, <32 x bfloat> %{{.*}} i1 false, i1 false)
- %r5.gfx1250 = rocdl.wmma.bf16.16x16x32.bf16 %arg16, %arg16, %arg17 {signA = false, signB = false, modC = 0 : i16} : (vector<16xbf16>, vector<16xbf16>, vector<32xbf16>) -> vector<32xbf16>
+ %r5.gfx1250 = rocdl.wmma.bf16.16x16x32.bf16 %arg16, %arg16, %arg17, modC = none : (vector<16xbf16>, vector<16xbf16>, vector<32xbf16>) -> vector<32xbf16>
// bf16 -> bf16 / f32
// CHECK: call <32 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v32bf16.v16bf16.v32f32(<16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}} i16 0, <32 x float> %{{.*}} i1 false, i1 false)
- %r6.gfx1250 = rocdl.wmma.bf16f32.16x16x32.bf16 %arg16, %arg16, %arg12 {modC = 0 : i16} : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xbf16>
+ %r6.gfx1250 = rocdl.wmma.bf16f32.16x16x32.bf16 %arg16, %arg16, %arg12, modC = none : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xbf16>
// f8/bf8 -> f16/f32
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r7.gfx1250 = rocdl.wmma.f32.16x16x64.fp8_fp8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r7.gfx1250 = rocdl.wmma.f32.16x16x64.fp8_fp8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r8.gfx1250 = rocdl.wmma.f32.16x16x64.fp8_bf8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r8.gfx1250 = rocdl.wmma.f32.16x16x64.fp8_bf8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r9.gfx1250 = rocdl.wmma.f32.16x16x64.bf8_fp8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r9.gfx1250 = rocdl.wmma.f32.16x16x64.bf8_fp8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r10.gfx1250 = rocdl.wmma.f32.16x16x64.bf8_bf8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r10.gfx1250 = rocdl.wmma.f32.16x16x64.bf8_bf8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r11.gfx1250 = rocdl.wmma.f16.16x16x64.fp8_fp8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r11.gfx1250 = rocdl.wmma.f16.16x16x64.fp8_fp8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r12.gfx1250 = rocdl.wmma.f16.16x16x64.fp8_bf8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r12.gfx1250 = rocdl.wmma.f16.16x16x64.fp8_bf8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r13.gfx1250 = rocdl.wmma.f16.16x16x64.bf8_fp8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r13.gfx1250 = rocdl.wmma.f16.16x16x64.bf8_fp8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r14.gfx1250 = rocdl.wmma.f16.16x16x64.bf8_bf8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r14.gfx1250 = rocdl.wmma.f16.16x16x64.bf8_bf8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r15.gfx1250 = rocdl.wmma.f32.16x16x128.fp8_fp8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r15.gfx1250 = rocdl.wmma.f32.16x16x128.fp8_fp8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r16.gfx1250 = rocdl.wmma.f32.16x16x128.fp8_bf8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r16.gfx1250 = rocdl.wmma.f32.16x16x128.fp8_bf8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r17.gfx1250 = rocdl.wmma.f32.16x16x128.bf8_fp8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r17.gfx1250 = rocdl.wmma.f32.16x16x128.bf8_fp8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v64f32.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x float> %{{.*}} i1 false, i1 false)
- %r18.gfx1250 = rocdl.wmma.f32.16x16x128.bf8_bf8 %arg5, %arg5, %arg13 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
+ %r18.gfx1250 = rocdl.wmma.f32.16x16x128.bf8_bf8 %arg5, %arg5, %arg13, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf32>) -> vector<64xf32>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r19.gfx1250 = rocdl.wmma.f16.16x16x128.fp8_fp8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r19.gfx1250 = rocdl.wmma.f16.16x16x128.fp8_fp8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r20.gfx1250 = rocdl.wmma.f16.16x16x128.fp8_bf8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r20.gfx1250 = rocdl.wmma.f16.16x16x128.fp8_bf8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r21.gfx1250 = rocdl.wmma.f16.16x16x128.bf8_fp8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r21.gfx1250 = rocdl.wmma.f16.16x16x128.bf8_fp8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// CHECK: call <64 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v64f16.v4i32(<4 x i32> %{{.*}} <4 x i32> %{{.*}} i16 0, <64 x half> %{{.*}} i1 false, i1 false)
- %r22.gfx1250 = rocdl.wmma.f16.16x16x128.bf8_bf8 %arg5, %arg5, %arg15 {signA = false, signB = false, modC = 0 : i16} : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
+ %r22.gfx1250 = rocdl.wmma.f16.16x16x128.bf8_bf8 %arg5, %arg5, %arg15, modC = none : (vector<4xi32>, vector<4xi32>, vector<64xf16>) -> vector<64xf16>
// iu8 -> i32
// CHECK: call <64 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v64i32.v4i32(i1 false, <4 x i32> %{{.*}} i1 false, <4 x i32> %{{.*}} <64 x i32> %{{.*}} i1 false, i1 false, i1 false)
@@ -1119,17 +1132,17 @@ llvm.func @rocdl.wmma(%arg0 : vector<8xf32>, %arg1 : vector<16 x f16>, %arg2 : v
// CHECK: call <64 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v64i32.v4i32(i1 true, <4 x i32> %{{.*}} i1 false, <4 x i32> %{{.*}} <64 x i32> %{{.*}} i1 true, i1 true, i1 false)
%r23b.gfx1250 = rocdl.wmma.i32.16x16x64.iu8 %arg5, %arg5, %arg14 {signA = true, signB = false, reuseA = true, reuseB = true, clamp=false} : (vector<4xi32>, vector<4xi32>, vector<64xi32>) -> vector<64xi32>
- // Test signA=false, signB=false with modC=1 for f32 gfx1250
+ // Test modC=1 for f32 gfx1250
// CHECK: call <4 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v4f32.v16f32(<16 x float> %{{.*}}, <16 x float> %{{.*}} i16 1, <4 x float> %{{.*}} i1 false, i1 false)
- %r1a.gfx1250 = rocdl.wmma.f32.16x16x4.f32 %arg10, %arg10, %arg11 {signA = false, signB = false, modC = 1 : i16, reuseA = false, reuseB = false} : (vector<16xf32>, vector<16xf32>, vector<4xf32>) -> vector<4xf32>
+ %r1a.gfx1250 = rocdl.wmma.f32.16x16x4.f32 %arg10, %arg10, %arg11, modC = neg : (vector<16xf32>, vector<16xf32>, vector<4xf32>) -> vector<4xf32>
- // Test with modC=2 and signA=false, signB=true, reuseA=true for f16 gfx1250
+ // Test with modC=2 and reuseA=true for f16 gfx1250
// CHECK: call <32 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v32f32.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}} i16 2, <32 x float> %{{.*}} i1 true, i1 false)
- %r2a.gfx1250 = rocdl.wmma.f32.16x16x32.f16 %arg1, %arg1, %arg12 {signA = false, signB = true, modC = 2 : i16, reuseA = true, reuseB = false} : (vector<16xf16>, vector<16xf16>, vector<32xf32>) -> vector<32xf32>
+ %r2a.gfx1250 = rocdl.wmma.f32.16x16x32.f16 %arg1, %arg1, %arg12, modC = abs {reuseA = true} : (vector<16xf16>, vector<16xf16>, vector<32xf32>) -> vector<32xf32>
- // Test with modC=3 and signA=false, signB=false, reuseB=true for bf16 gfx1250
+ // Test with modC=3 and reuseB=true for bf16 gfx1250
// CHECK: call <32 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v32f32.v16bf16(<16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}} i16 3, <32 x float> %{{.*}} i1 false, i1 true)
- %r3a.gfx1250 = rocdl.wmma.f32.16x16x32.bf16 %arg16, %arg16, %arg12 {signA = false, signB = false, modC = 3 : i16, reuseA = false, reuseB = true} : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xf32>
+ %r3a.gfx1250 = rocdl.wmma.f32.16x16x32.bf16 %arg16, %arg16, %arg12, modC = neg_abs {reuseB = true} : (vector<16xbf16>, vector<16xbf16>, vector<32xf32>) -> vector<32xf32>
// ---- Wave64 -----
@@ -1401,7 +1414,7 @@ llvm.func @rocdl.tensor.load.to.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<
%dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>,
%dgroup4 : vector<8xi32>) {
// CHECK: call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> %{{.*}}, i32 0)
- rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4, 0 : vector<4xi32>, vector<8xi32>
llvm.return
}
@@ -1410,7 +1423,7 @@ llvm.func @rocdl.tensor.store.from.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vect
%dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>,
%dgroup4 : vector<8xi32>) {
// CHECK: call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %{{.*}}, <8 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> %{{.*}}, i32 0)
- rocdl.tensor.store.from.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+ rocdl.tensor.store.from.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3, %dgroup4, 0 : vector<4xi32>, vector<8xi32>
llvm.return
}
@@ -1511,13 +1524,13 @@ llvm.func @rocdl.raw.ptr.buffer(%rsrc : !llvm.ptr<8>,
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : i32
- %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xi32>
- %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xi32>
+ %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : i32
+ %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xi32>
+ %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xi32>
- rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : i32
- rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xi32>
- rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, %aux : vector<4xi32>
+ rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xi32>
+ rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %soffset, 0 : vector<4xi32>
llvm.return
}
@@ -1526,10 +1539,9 @@ llvm.func @rocdl.raw.ptr.buffer.load.lds(%rsrc : !llvm.ptr<8>, %dstLds : !llvm.p
%voffset : i32, %soffset : i32) {
%size = llvm.mlir.constant(4 : i32) : i32
%offset = llvm.mlir.constant(128 : i32) : i32
- %aux = llvm.mlir.constant(1 : i32) : i32
// CHECK-LABEL: rocdl.raw.ptr.buffer.load.lds
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.load.lds(ptr addrspace(8) %{{.*}}, ptr addrspace(3) %{{.*}}, i32 4, i32 %{{.*}}, i32 %{{.*}}, i32 128, i32 1
- rocdl.raw.ptr.buffer.load.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, %aux
+ rocdl.raw.ptr.buffer.load.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, pre_gfx12<glc>
llvm.return
}
@@ -1538,10 +1550,9 @@ llvm.func @rocdl.raw.ptr.buffer.load.async.lds(%rsrc : !llvm.ptr<8>, %dstLds : !
%voffset : i32, %soffset : i32) {
%size = llvm.mlir.constant(4 : i32) : i32
%offset = llvm.mlir.constant(128 : i32) : i32
- %aux = llvm.mlir.constant(1 : i32) : i32
// CHECK-LABEL: rocdl.raw.ptr.buffer.load.async.lds
// CHECK: call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %{{.*}}, ptr addrspace(3) %{{.*}}, i32 4, i32 %{{.*}}, i32 %{{.*}}, i32 128, i32 1
- rocdl.raw.ptr.buffer.load.async.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, %aux
+ rocdl.raw.ptr.buffer.load.async.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, pre_gfx12<glc>
llvm.return
}
@@ -1549,14 +1560,16 @@ llvm.func @rocdl.raw.ptr.buffer.load.async.lds(%rsrc : !llvm.ptr<8>, %dstLds : !
llvm.func @rocdl.global.prefetch(%ptr : !llvm.ptr<1>) {
// CHECK-LABEL: rocdl.global.prefetch
// CHECK: call void @llvm.amdgcn.global.prefetch(ptr addrspace(1) %{{.*}}, i32 0)
- rocdl.global.prefetch %ptr, scope 0 : !llvm.ptr<1>
+ rocdl.global.prefetch %ptr, 0 : !llvm.ptr<1>
+ // CHECK: call void @llvm.amdgcn.global.prefetch(ptr addrspace(1) %{{.*}}, i32 12)
+ rocdl.global.prefetch %ptr, gfx12<nt_rt|scope_se> : !llvm.ptr<1>
llvm.return
}
llvm.func @rocdl.flat.prefetch(%ptr : !llvm.ptr) {
// CHECK-LABEL: rocdl.flat.prefetch
// CHECK: call void @llvm.amdgcn.flat.prefetch(ptr %{{.*}}, i32 0)
- rocdl.flat.prefetch %ptr, scope 0 : !llvm.ptr
+ rocdl.flat.prefetch %ptr, 0 : !llvm.ptr
llvm.return
}
@@ -1576,101 +1589,78 @@ llvm.func @rocdl.wmma.scale(%arg0: i32, %arg1: vector<4xf32>, %arg2: vector<8xi3
// Test with default attributes (all zeros/false)
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 0, <16 x i32> %{{.*}}, i32 0, <16 x i32> %{{.*}}, i16 0, <4 x float> %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i1 false, i1 false)
- %r00 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0
- {fmtA = 0 : i32, fmtB = 0 : i32, modC = 0 : i16,
- scaleAType = 0 : i32, fmtScaleA = 0 : i32,
- scaleBType = 0 : i32, fmtScaleB = 0 : i32,
- reuseA = false, reuseB = false} :
+ %r00 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0,
+ fmtA = fp8_e4m3, fmtB = fp8_e4m3, modC = none,
+ scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with
diff erent matrix formats (FP8 x BF8)
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 0, <16 x i32> %{{.*}}, i32 1, <16 x i32> %{{.*}}, i16 0, <4 x float> %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i1 false, i1 false)
- %r01 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0
- {fmtA = 0 : i32, fmtB = 1 : i32, modC = 0 : i16,
- scaleAType = 1 : i32, fmtScaleA = 1 : i32,
- scaleBType = 1 : i32, fmtScaleB = 1 : i32,
- reuseA = false, reuseB = false} :
+ %r01 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0,
+ fmtA = fp8_e4m3, fmtB = fp8_e5m2, modC = none,
+ scaleAType = row1, fmtScaleA = e5m3, scaleBType = row1, fmtScaleB = e5m3 :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with FP8 x FP6 (
diff erent vector sizes) and modC = 1 (negate)
- // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v12i32(i32 0, <16 x i32> %{{.*}}, i32 2, <12 x i32> %{{.*}}, i16 1, <4 x float> %{{.*}}, i32 2, i32 2, i32 %{{.*}}, i32 2, i32 2, i32 %{{.*}}, i1 false, i1 false)
- %r02 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg3, %arg1, %arg0, %arg0
- {fmtA = 0 : i32, fmtB = 2 : i32, modC = 1 : i16,
- scaleAType = 2 : i32, fmtScaleA = 2 : i32,
- scaleBType = 2 : i32, fmtScaleB = 2 : i32,
- reuseA = false, reuseB = false} :
+ // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v12i32(i32 0, <16 x i32> %{{.*}}, i32 2, <12 x i32> %{{.*}}, i16 1, <4 x float> %{{.*}}, i32 1, i32 2, i32 %{{.*}}, i32 1, i32 2, i32 %{{.*}}, i1 false, i1 false)
+ %r02 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg3, %arg1, %arg0, %arg0,
+ fmtA = fp8_e4m3, fmtB = fp6_e2m3, modC = neg,
+ scaleAType = row1, fmtScaleA = e4m3, scaleBType = row1, fmtScaleB = e4m3 :
(vector<16xi32>, vector<12xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with BF8 x BF6 and modC = 2 (abs)
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v12i32(i32 1, <16 x i32> %{{.*}}, i32 3, <12 x i32> %{{.*}}, i16 2, <4 x float> %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i1 false, i1 false)
- %r03 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg3, %arg1, %arg0, %arg0
- {fmtA = 1 : i32, fmtB = 3 : i32, modC = 2 : i16,
- scaleAType = 0 : i32, fmtScaleA = 0 : i32,
- scaleBType = 0 : i32, fmtScaleB = 0 : i32,
- reuseA = false, reuseB = false} :
+ %r03 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg3, %arg1, %arg0, %arg0,
+ fmtA = fp8_e5m2, fmtB = fp6_e3m2, modC = abs,
+ scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 :
(vector<16xi32>, vector<12xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with FP8 x FP4 and modC = 3 (negate(abs))
- // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v8i32(i32 0, <16 x i32> %{{.*}}, i32 4, <8 x i32> %{{.*}}, i16 3, <4 x float> %{{.*}}, i32 3, i32 3, i32 %{{.*}}, i32 3, i32 3, i32 %{{.*}}, i1 false, i1 false)
- %r04 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg2, %arg1, %arg0, %arg0
- {fmtA = 0 : i32, fmtB = 4 : i32, modC = 3 : i16,
- scaleAType = 3 : i32, fmtScaleA = 3 : i32,
- scaleBType = 3 : i32, fmtScaleB = 3 : i32,
- reuseA = false, reuseB = false} :
+ // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v8i32(i32 0, <16 x i32> %{{.*}}, i32 4, <8 x i32> %{{.*}}, i16 3, <4 x float> %{{.*}}, i32 1, i32 2, i32 %{{.*}}, i32 1, i32 2, i32 %{{.*}}, i1 false, i1 false)
+ %r04 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg2, %arg1, %arg0, %arg0,
+ fmtA = fp8_e4m3, fmtB = fp4_e2m1, modC = neg_abs,
+ scaleAType = row1, fmtScaleA = e4m3, scaleBType = row1, fmtScaleB = e4m3 :
(vector<16xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with reuseA = true
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 2, <16 x i32> %{{.*}}, i32 2, <16 x i32> %{{.*}}, i16 0, <4 x float> %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i1 true, i1 false)
- %r10 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0
- {fmtA = 2 : i32, fmtB = 2 : i32, modC = 0 : i16,
- scaleAType = 0 : i32, fmtScaleA = 0 : i32,
- scaleBType = 0 : i32, fmtScaleB = 0 : i32,
- reuseA = true, reuseB = false} :
+ %r10 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0,
+ fmtA = fp6_e2m3, fmtB = fp6_e2m3, modC = none,
+ scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 {reuseA = true} :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with reuseB = true
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 3, <16 x i32> %{{.*}}, i32 3, <16 x i32> %{{.*}}, i16 0, <4 x float> %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i32 0, i32 0, i32 %{{.*}}, i1 false, i1 true)
- %r11 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0
- {fmtA = 3 : i32, fmtB = 3 : i32, modC = 0 : i16,
- scaleAType = 0 : i32, fmtScaleA = 0 : i32,
- scaleBType = 0 : i32, fmtScaleB = 0 : i32,
- reuseA = false, reuseB = true} :
+ %r11 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0,
+ fmtA = fp6_e3m2, fmtB = fp6_e3m2, modC = none,
+ scaleAType = row0, fmtScaleA = e8, scaleBType = row0, fmtScaleB = e8 {reuseB = true} :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test with both reuseA and reuseB = true
// CHECK: call <4 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 4, <16 x i32> %{{.*}}, i32 4, <16 x i32> %{{.*}}, i16 1, <4 x float> %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i1 true, i1 true)
- %r12 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0
- {fmtA = 4 : i32, fmtB = 4 : i32, modC = 1 : i16,
- scaleAType = 1 : i32, fmtScaleA = 1 : i32,
- scaleBType = 1 : i32, fmtScaleB = 1 : i32,
- reuseA = true, reuseB = true} :
+ %r12 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg0, %arg0,
+ fmtA = fp4_e2m1, fmtB = fp4_e2m1, modC = neg,
+ scaleAType = row1, fmtScaleA = e5m3, scaleBType = row1, fmtScaleB = e5m3 {reuseA = true, reuseB = true} :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>
// Test scale16 variant with i64 scale exponents
- // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 0, <16 x i32> %{{.*}}, i32 1, <16 x i32> %{{.*}}, i16 2, <4 x float> %{{.*}}, i32 2, i32 2, i64 %{{.*}}, i32 2, i32 2, i64 %{{.*}}, i1 false, i1 false)
- %r_scale16 = rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg8, %arg8
- {fmtA = 0 : i32, fmtB = 1 : i32, modC = 2 : i16,
- scaleAType = 2 : i32, fmtScaleA = 2 : i32,
- scaleBType = 2 : i32, fmtScaleB = 2 : i32,
- reuseA = false, reuseB = false} :
+ // CHECK: call <4 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v4f32.v16i32.v16i32(i32 0, <16 x i32> %{{.*}}, i32 1, <16 x i32> %{{.*}}, i16 2, <4 x float> %{{.*}}, i32 1, i32 2, i64 %{{.*}}, i32 1, i32 2, i64 %{{.*}}, i1 false, i1 false)
+ %r_scale16 = rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %arg5, %arg5, %arg1, %arg8, %arg8,
+ fmtA = fp8_e4m3, fmtB = fp8_e5m2, modC = abs,
+ scaleAType = row1, fmtScaleA = e4m3, scaleBType = row1, fmtScaleB = e4m3 :
(vector<16xi32>, vector<16xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>
// Test f4 variant (no matrix format parameters)
// CHECK: call <8 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v8f32.v16i32.v8i32(<16 x i32> %{{.*}}, <8 x i32> %{{.*}}, i16 0, <8 x float> %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i32 1, i32 1, i32 %{{.*}}, i1 false, i1 false)
- %r_f4 = rocdl.wmma.scale.f32.32x16x128.f4 %arg5, %arg2, %arg9, %arg0, %arg0
- {modC = 0 : i16,
- scaleAType = 1 : i32, fmtScaleA = 1 : i32,
- scaleBType = 1 : i32, fmtScaleB = 1 : i32,
- reuseA = false, reuseB = false} :
+ %r_f4 = rocdl.wmma.scale.f32.32x16x128.f4 %arg5, %arg2, %arg9, %arg0, %arg0,
+ modC = none, scaleAType = row1, fmtScaleA = e5m3, scaleBType = row1, fmtScaleB = e5m3 :
(vector<16xi32>, vector<8xi32>, vector<8xf32>, i32, i32) -> vector<8xf32>
// Test f4 scale16 variant with varied attributes
- // CHECK: call <8 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v8f32.v16i32.v8i32(<16 x i32> %{{.*}}, <8 x i32> %{{.*}}, i16 3, <8 x float> %{{.*}}, i32 2, i32 3, i64 %{{.*}}, i32 3, i32 2, i64 %{{.*}}, i1 true, i1 true)
- %r_f4_scale16 = rocdl.wmma.scale16.f32.32x16x128.f4 %arg5, %arg2, %arg9, %arg8, %arg8
- {modC = 3 : i16,
- scaleAType = 2 : i32, fmtScaleA = 3 : i32,
- scaleBType = 3 : i32, fmtScaleB = 2 : i32,
- reuseA = true, reuseB = true} :
+ // CHECK: call <8 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v8f32.v16i32.v8i32(<16 x i32> %{{.*}}, <8 x i32> %{{.*}}, i16 3, <8 x float> %{{.*}}, i32 0, i32 2, i64 %{{.*}}, i32 1, i32 2, i64 %{{.*}}, i1 true, i1 true)
+ %r_f4_scale16 = rocdl.wmma.scale16.f32.32x16x128.f4 %arg5, %arg2, %arg9, %arg8, %arg8,
+ modC = neg_abs, scaleAType = row0, fmtScaleA = e4m3, scaleBType = row1, fmtScaleB = e4m3
+ {reuseA = true, reuseB = true} :
(vector<16xi32>, vector<8xi32>, vector<8xf32>, i64, i64) -> vector<8xf32>
llvm.return %r00 : vector<4xf32>
@@ -1685,8 +1675,8 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.f32(%rsrc : !llvm.ptr<8>,
// CHECK: [[PTR_FMAX:%.+]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret float [[PTR_FMAX]]
- %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32
- %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32
+ %fadd = rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %fmax = rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
llvm.return %fmax : f32
}
@@ -1700,8 +1690,8 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.i32(%rsrc : !llvm.ptr<8>,
// CHECK: [[PTR_UMIN:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret i32 [[PTR_UMIN]]
- %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32
+ %smax = rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %umin = rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
llvm.return %umin : i32
}
@@ -1714,7 +1704,7 @@ llvm.func @rocdl.raw.ptr.buffer.atomic.cmpswap(%rsrc : !llvm.ptr<8>,
// CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, ptr addrspace(8) %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret i32 [[val]]
- %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, %aux : i32
+ %val = rocdl.raw.ptr.buffer.atomic.cmpswap %src, %cmp, %rsrc, %offset, %soffset, 0 : i32
llvm.return %val : i32
}
@@ -1728,18 +1718,20 @@ llvm.func @rocdl.raw.buffer(%rsrc : vector<4xi32>,
// CHECK: call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call <2 x i32> @llvm.amdgcn.raw.buffer.load.v2i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+ // CHECK: call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 3)
// CHECK: call void @llvm.amdgcn.raw.buffer.store.i32(i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32> %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: call void @llvm.amdgcn.raw.buffer.store.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
- %r1 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : i32
- %r2 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xi32>
- %r4 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xi32>
+ %r1 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : i32
+ %r2 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : vector<2xi32>
+ %r4 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, 0 : vector<4xi32>
+ %r942 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, gfx942<sc0|nt> : i32
- rocdl.raw.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : i32
- rocdl.raw.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xi32>
- rocdl.raw.buffer.store %vdata4, %rsrc, %offset, %soffset, %aux : vector<4xi32>
+ rocdl.raw.buffer.store %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ rocdl.raw.buffer.store %vdata2, %rsrc, %offset, %soffset, 0 : vector<2xi32>
+ rocdl.raw.buffer.store %vdata4, %rsrc, %offset, %soffset, 0 : vector<4xi32>
llvm.return
}
@@ -1753,8 +1745,8 @@ llvm.func @rocdl.raw.buffer.atomic.f32(%rsrc : vector<4xi32>,
// CHECK: [[RAW_FMAX:%.+]] = call float @llvm.amdgcn.raw.buffer.atomic.fmax.f32(float %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret float [[RAW_FMAX]]
- %fadd = rocdl.raw.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32
- %fmax = rocdl.raw.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32
+ %fadd = rocdl.raw.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, 0 : f32
+ %fmax = rocdl.raw.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, 0 : f32
llvm.return %fmax : f32
}
@@ -1768,8 +1760,8 @@ llvm.func @rocdl.raw.buffer.atomic.i32(%rsrc : vector<4xi32>,
// CHECK: [[RAW_UMIN:%.+]] = call i32 @llvm.amdgcn.raw.buffer.atomic.umin.i32(i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret i32 [[RAW_UMIN]]
- %smax = rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32
- %umin = rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32
+ %smax = rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, 0 : i32
+ %umin = rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, 0 : i32
llvm.return %umin : i32
}
@@ -1782,7 +1774,7 @@ llvm.func @rocdl.raw.buffer.atomic.cmpswap(%rsrc : vector<4xi32>,
// CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
// CHECK: ret i32 [[val]]
- %val = rocdl.raw.buffer.atomic.cmpswap(%src, %cmp, %rsrc, %offset, %soffset, %aux) : i32, vector<4xi32>
+ %val = rocdl.raw.buffer.atomic.cmpswap(%src, %cmp, %rsrc, %offset, %soffset, 0) : i32, vector<4xi32>
llvm.return %val : i32
}
More information about the Mlir-commits
mailing list