[Mlir-commits] [mlir] [MLIR][NVVM] Add support for narrow-fp to bf16x2 conversions (PR #200157)
Srinivasa Ravi
llvmlistbot at llvm.org
Mon Jun 8 03:32:47 PDT 2026
https://github.com/Wolfram70 updated https://github.com/llvm/llvm-project/pull/200157
>From 54ea0edf0a9484b84ca3b32e1753404e1c3617d8 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Thu, 28 May 2026 10:13:51 +0000
Subject: [PATCH 1/5] [MLIR][NVVM] Add support for narrow-fp to bf16x2
conversions
This change adds the following NVVM Ops to support narrow-fp to bf16x2
conversions:
- `nvvm.convert.f6x2.to.bf16x2`
- `nvvm.convert.f4x2.to.bf16x2`
- `nvvm.convert.f8x2.to.bf16x2` (updated to allow `E4M3FN` and `E5M2` types)
Also removes unnecessary verifiers for narrow-fp to `f16x2` conversions
to instead use `TypeAttrOf` to validate the source type in the ODS
definition.
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 48 +++--
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 170 +++++++++++++-----
.../Target/LLVMIR/nvvm/convert_fp4x2.mlir | 22 +++
.../Target/LLVMIR/nvvm/convert_fp6x2.mlir | 42 +++++
.../Target/LLVMIR/nvvm/convert_fp8x2.mlir | 40 +++++
mlir/test/Target/LLVMIR/nvvmir-invalid.mlir | 50 +++++-
6 files changed, 312 insertions(+), 60 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 6230c0f91a865..b77cae747af39 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2176,7 +2176,7 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
}];
}
-class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstType>
+class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstType, list<Type> supportedTypes, int needVerify = 0>
: NVVM_SingleResultIntrinsicOp<"convert." # !tolower(srcType) # "x2.to." # !tolower(dstType) # "x2", [], "$dst"> {
let summary = "Convert a pair of " # !tolower(srcType) # " inputs to " # !tolower(dstType) # "x2";
let description = [{
@@ -2185,32 +2185,52 @@ class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstTy
!tolower(dstType) # [{.
The result `dst` is represented as a vector of }] # !tolower(dstType) # [{ elements.
- }] #
- !if(!eq(dstType, "F16"),
- [{The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction."}], "") # [{
+
+ The `relu` attribute, when set, lowers to the '.relu' variant of
+ the cvt instruction.}] #
+
+ !if(!eq(dstType, "BF16"),
+ [{The `sat` attribute specifies the saturation mode.
+
+ The optional scaling-factors for each of the inputs are provided through
+ the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported
+ as the type of the scale-factor currently.}], "") # [{
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
let results = (outs VectorOfLengthAndType<[2], [!cast<Type>(dstType)]>:$dst);
let arguments = !if(!eq(dstType, "F16"),
(ins srcArgType:$src,
- DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$srcType),
+ TypeAttrOf<AnyTypeOf<supportedTypes>>:$srcType,
+ DefaultValuedAttr<BoolAttr, "false">:$relu),
(ins srcArgType:$src,
- TypeAttr:$srcType));
- let assemblyFormat = "$src attr-dict `:` type($src) `(` $srcType `)` `->` type($dst)";
- let hasVerifier = 1;
+ Optional<I16>:$scaleFactor,
+ TypeAttrOf<AnyTypeOf<supportedTypes>>:$srcType,
+ DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
+ DefaultValuedAttr<BoolAttr, "false">:$relu));
+ let assemblyFormat =
+ !if(!eq(dstType, "F16"),
+ "$src attr-dict `:` type($src) `(` $srcType `)` `->` type($dst)",
+ "$src (`,` $scaleFactor^)? attr-dict `:` type($src) `(` $srcType `)` `->` type($dst)");
+ let hasVerifier = needVerify;
}
def NVVM_ConvertF8x2ToF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "F16">;
+ NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "F16",
+ [F8E4M3FN, F8E5M2]>;
def NVVM_ConvertF8x2ToBF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "BF16">;
+ NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "BF16",
+ [F8E8M0FNU, F8E4M3FN, F8E5M2], 1>;
def NVVM_ConvertF6x2ToF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "F16">;
+ NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "F16",
+ [F6E2M3FN, F6E3M2FN]>;
+def NVVM_ConvertF6x2ToBF16x2Op :
+ NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "BF16",
+ [F6E2M3FN, F6E3M2FN]>;
def NVVM_ConvertF4x2ToF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F4", I8, "F16">;
+ NVVM_ConvertToFP16x2Op_Base<"F4", I8, "F16", [F4E2M1FN]>;
+def NVVM_ConvertF4x2ToBF16x2Op :
+ NVVM_ConvertToFP16x2Op_Base<"F4", I8, "BF16", [F4E2M1FN]>;
def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> {
let summary = "Convert a pair of f32 inputs to S2F6x2";
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index d246ce2651ef7..17027b5622d79 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -542,47 +542,20 @@ LogicalResult ConvertF32x2ToF4x2Op::verify() {
return success();
}
-LogicalResult ConvertF8x2ToF16x2Op::verify() {
- mlir::MLIRContext *ctx = getContext();
-
- if (!llvm::isa<Float8E4M3FNType, Float8E5M2Type>(getSrcType()))
- return emitOpError("Only ")
- << mlir::Float8E4M3FNType::get(ctx) << " and "
- << mlir::Float8E5M2Type::get(ctx)
- << " types are supported for conversions from f8x2 to f16x2.";
-
- return success();
-}
-
LogicalResult ConvertF8x2ToBF16x2Op::verify() {
mlir::MLIRContext *ctx = getContext();
- if (!llvm::isa<Float8E8M0FNUType>(getSrcType()))
- return emitOpError("Only ")
- << mlir::Float8E8M0FNUType::get(ctx)
- << " type is supported for conversions from f8x2 to bf16x2.";
-
- return success();
-}
-
-LogicalResult ConvertF6x2ToF16x2Op::verify() {
- mlir::MLIRContext *ctx = getContext();
-
- if (!llvm::isa<Float6E2M3FNType, Float6E3M2FNType>(getSrcType()))
- return emitOpError("Only ")
- << mlir::Float6E2M3FNType::get(ctx) << " and "
- << mlir::Float6E3M2FNType::get(ctx)
- << " types are supported for conversions from f6x2 to f16x2.";
-
- return success();
-}
-
-LogicalResult ConvertF4x2ToF16x2Op::verify() {
- mlir::MLIRContext *ctx = getContext();
-
- if (!llvm::isa<Float4E2M1FNType>(getSrcType()))
- return emitOpError("Only ")
- << mlir::Float4E2M1FNType::get(ctx)
- << " type is supported for conversions from f4x2 to f16x2.";
+ if (llvm::isa<Float8E8M0FNUType>(getSrcType())) {
+ if (getSat() != SaturationMode::NONE)
+ return emitOpError(
+ "Only NONE saturation mode is supported for conversions from ")
+ << Float8E8M0FNUType::get(ctx) << " type";
+ if (getScaleFactor())
+ return emitOpError("scaleFactor not supported for conversions from ")
+ << Float8E8M0FNUType::get(ctx) << " type";
+ if (getRelu())
+ return emitOpError("relu not supported for conversions from ")
+ << Float8E8M0FNUType::get(ctx) << " type";
+ }
return success();
}
@@ -4816,13 +4789,50 @@ NVVM::IDArgPair ConvertF8x2ToF16x2Op::getIntrinsicIDAndArgs(
NVVM::IDArgPair ConvertF8x2ToBF16x2Op::getIntrinsicIDAndArgs(
Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
auto curOp = cast<NVVM::ConvertF8x2ToBF16x2Op>(op);
+ bool hasScale = static_cast<bool>(curOp.getScaleFactor());
+ bool hasSatfinite = curOp.getSat() == NVVM::SaturationMode::SATFINITE;
+ bool hasRelu = curOp.getRelu();
+
+ static constexpr llvm::Intrinsic::ID E4M3Ids[] = {
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ };
+
+ static constexpr llvm::Intrinsic::ID E5M2Ids[] = {
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ };
- llvm::Intrinsic::ID intId = llvm::Intrinsic::nvvm_ue8m0x2_to_bf16x2;
+ llvm::Intrinsic::ID intId =
+ llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(curOp.getSrcType())
+ .Case([&](Float8E8M0FNUType type) {
+ return llvm::Intrinsic::nvvm_ue8m0x2_to_bf16x2;
+ })
+ .Case([&](Float8E4M3FNType type) {
+ return E4M3Ids[hasSatfinite << 1 | hasRelu];
+ })
+ .Case([&](Float8E5M2Type type) {
+ return E5M2Ids[hasSatfinite << 1 | hasRelu];
+ })
+ .Default([](mlir::Type type) {
+ llvm_unreachable("Invalid type for ConvertF8x2ToBF16x2Op");
+ return llvm::Intrinsic::not_intrinsic;
+ });
llvm::Value *packedI16 =
builder.CreateBitCast(mt.lookupValue(curOp.getSrc()),
llvm::Type::getInt16Ty(builder.getContext()));
- return {intId, {packedI16}};
+ llvm::SmallVector<llvm::Value *> args;
+ args.push_back(packedI16);
+ if (!isa<Float8E8M0FNUType>(curOp.getSrcType()))
+ args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(0x7f7f));
+
+ return {intId, std::move(args)};
}
NVVM::IDArgPair ConvertF6x2ToF16x2Op::getIntrinsicIDAndArgs(
@@ -4853,6 +4863,49 @@ NVVM::IDArgPair ConvertF6x2ToF16x2Op::getIntrinsicIDAndArgs(
return {intId, {packedI16}};
}
+NVVM::IDArgPair ConvertF6x2ToBF16x2Op::getIntrinsicIDAndArgs(
+ Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
+ auto curOp = cast<NVVM::ConvertF6x2ToBF16x2Op>(op);
+ bool hasScale = static_cast<bool>(curOp.getScaleFactor());
+ bool hasSatfinite = curOp.getSat() == NVVM::SaturationMode::SATFINITE;
+ bool hasRelu = curOp.getRelu();
+
+ static constexpr llvm::Intrinsic::ID E2M3Ids[] = {
+ llvm::Intrinsic::nvvm_e2m3x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m3x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m3x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m3x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ };
+
+ static constexpr llvm::Intrinsic::ID E3M2Ids[] = {
+ llvm::Intrinsic::nvvm_e3m2x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e3m2x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e3m2x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e3m2x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ };
+
+ unsigned idx = (hasSatfinite << 1) | hasRelu;
+ llvm::Intrinsic::ID intId =
+ llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(curOp.getSrcType())
+ .Case([&](Float6E2M3FNType type) { return E2M3Ids[idx]; })
+ .Case([&](Float6E3M2FNType type) { return E3M2Ids[idx]; })
+ .Default([](mlir::Type type) {
+ llvm_unreachable("Invalid type for ConvertF6x2ToBF16x2Op");
+ return llvm::Intrinsic::not_intrinsic;
+ });
+
+ llvm::Value *packedI16 =
+ builder.CreateBitCast(mt.lookupValue(curOp.getSrc()),
+ llvm::Type::getInt16Ty(builder.getContext()));
+
+ llvm::SmallVector<llvm::Value *> args;
+ args.push_back(packedI16);
+ args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(0x7f7f));
+
+ return {intId, std::move(args)};
+}
+
NVVM::IDArgPair ConvertF4x2ToF16x2Op::getIntrinsicIDAndArgs(
Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
auto curOp = cast<NVVM::ConvertF4x2ToF16x2Op>(op);
@@ -4877,6 +4930,41 @@ NVVM::IDArgPair ConvertF4x2ToF16x2Op::getIntrinsicIDAndArgs(
return {intId, {extendedI16}};
}
+NVVM::IDArgPair ConvertF4x2ToBF16x2Op::getIntrinsicIDAndArgs(
+ Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
+ auto curOp = cast<NVVM::ConvertF4x2ToBF16x2Op>(op);
+ bool hasScale = static_cast<bool>(curOp.getScaleFactor());
+ bool hasSatfinite = curOp.getSat() == NVVM::SaturationMode::SATFINITE;
+ bool hasRelu = curOp.getRelu();
+
+ static constexpr llvm::Intrinsic::ID E2M1Ids[] = {
+ llvm::Intrinsic::nvvm_e2m1x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m1x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m1x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e2m1x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ };
+
+ unsigned idx = (hasSatfinite << 1) | hasRelu;
+ llvm::Intrinsic::ID intId =
+ llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(curOp.getSrcType())
+ .Case([&](Float4E2M1FNType type) { return E2M1Ids[idx]; })
+ .Default([](mlir::Type type) {
+ llvm_unreachable("Invalid type for ConvertF4x2ToBF16x2Op");
+ return llvm::Intrinsic::not_intrinsic;
+ });
+
+ llvm::Value *extendedI16 =
+ builder.CreateZExt(mt.lookupValue(curOp.getSrc()),
+ llvm::Type::getInt16Ty(builder.getContext()));
+
+ llvm::SmallVector<llvm::Value *> args;
+ args.push_back(extendedI16);
+ args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(0x7f7f));
+
+ return {intId, std::move(args)};
+}
+
NVVM::IDArgPair ConvertF32x2ToS2F6x2Op::getIntrinsicIDAndArgs(
Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
auto thisOp = cast<NVVM::ConvertF32x2ToS2F6x2Op>(op);
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
index 3d3bd714fa8fa..fc25a194ef236 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
@@ -49,3 +49,25 @@ llvm.func @convert_f4x2_to_f16x2(%src : i8) {
%res2 = nvvm.convert.f4x2.to.f16x2 %src {relu = true} : i8 (f4E2M1FN)-> vector<2xf16>
llvm.return
}
+
+// -----
+
+// CHECK-LABEL: @convert_f4x2_to_bf16x2
+llvm.func @convert_f4x2_to_bf16x2(%src : i8, %scale_factor : i16) {
+ // CHECK: %[[res1:.*]] = zext i8 %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m1x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res1]], i16 32639)
+ %res1 = nvvm.convert.f4x2.to.bf16x2 %src : i8 (f4E2M1FN) -> vector<2xbf16>
+ // CHECK: %[[res2:.*]] = zext i8 %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m1x2.to.bf16x2.rn.relu.scale.n2.ue8m0(i16 %[[res2]], i16 32639)
+ %res2 = nvvm.convert.f4x2.to.bf16x2 %src {relu = true} : i8 (f4E2M1FN) -> vector<2xbf16>
+ // CHECK: %[[res3:.*]] = zext i8 %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m1x2.to.bf16x2.rn.satfinite.scale.n2.ue8m0(i16 %[[res3]], i16 32639)
+ %res3 = nvvm.convert.f4x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : i8 (f4E2M1FN) -> vector<2xbf16>
+ // CHECK: %[[res4:.*]] = zext i8 %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m1x2.to.bf16x2.rn.relu.satfinite.scale.n2.ue8m0(i16 %[[res4]], i16 32639)
+ %res4 = nvvm.convert.f4x2.to.bf16x2 %src {relu = true, sat = #nvvm.sat_mode<satfinite>} : i8 (f4E2M1FN) -> vector<2xbf16>
+ // CHECK: %[[res5:.*]] = zext i8 %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m1x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res5]], i16 %{{.*}})
+ %res5 = nvvm.convert.f4x2.to.bf16x2 %src, %scale_factor : i8 (f4E2M1FN) -> vector<2xbf16>
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
index 8d9e5ff2a6a82..e83f4fe6449db 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
@@ -114,3 +114,45 @@ llvm.func @convert_f6x2_to_f16x2_e3m2(%src : vector<2xi8>) {
%res2 = nvvm.convert.f6x2.to.f16x2 %src {relu = true} : vector<2xi8> (f6E3M2FN)-> vector<2xf16>
llvm.return
}
+
+// -----
+
+// CHECK-LABEL: @convert_f6x2_to_bf16x2_e2m3
+llvm.func @convert_f6x2_to_bf16x2_e2m3(%src : vector<2xi8>, %scale_factor : i16) {
+ // CHECK: %[[res1:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m3x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res1]], i16 32639)
+ %res1 = nvvm.convert.f6x2.to.bf16x2 %src : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ // CHECK: %[[res2:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m3x2.to.bf16x2.rn.relu.scale.n2.ue8m0(i16 %[[res2]], i16 32639)
+ %res2 = nvvm.convert.f6x2.to.bf16x2 %src {relu = true} : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ // CHECK: %[[res3:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m3x2.to.bf16x2.rn.satfinite.scale.n2.ue8m0(i16 %[[res3]], i16 32639)
+ %res3 = nvvm.convert.f6x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ // CHECK: %[[res4:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m3x2.to.bf16x2.rn.relu.satfinite.scale.n2.ue8m0(i16 %[[res4]], i16 32639)
+ %res4 = nvvm.convert.f6x2.to.bf16x2 %src {relu = true, sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ // CHECK: %[[res5:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e2m3x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res5]], i16 %{{.*}})
+ %res5 = nvvm.convert.f6x2.to.bf16x2 %src, %scale_factor : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f6x2_to_bf16x2_e3m2
+llvm.func @convert_f6x2_to_bf16x2_e3m2(%src : vector<2xi8>, %scale_factor : i16) {
+ // CHECK: %[[res1:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e3m2x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res1]], i16 32639)
+ %res1 = nvvm.convert.f6x2.to.bf16x2 %src : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+ // CHECK: %[[res2:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e3m2x2.to.bf16x2.rn.relu.scale.n2.ue8m0(i16 %[[res2]], i16 32639)
+ %res2 = nvvm.convert.f6x2.to.bf16x2 %src {relu = true} : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+ // CHECK: %[[res3:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e3m2x2.to.bf16x2.rn.satfinite.scale.n2.ue8m0(i16 %[[res3]], i16 32639)
+ %res3 = nvvm.convert.f6x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+ // CHECK: %[[res4:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e3m2x2.to.bf16x2.rn.relu.satfinite.scale.n2.ue8m0(i16 %[[res4]], i16 32639)
+ %res4 = nvvm.convert.f6x2.to.bf16x2 %src {relu = true, sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+ // CHECK: %[[res5:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e3m2x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res5]], i16 %{{.*}})
+ %res5 = nvvm.convert.f6x2.to.bf16x2 %src, %scale_factor : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
index d8002d790b6a2..317e95dc3a75b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
@@ -159,3 +159,43 @@ llvm.func @convert_f8x2_to_bf16x2_ue8m0(%src : vector<2xi8>) {
%res1 = nvvm.convert.f8x2.to.bf16x2 %src : vector<2xi8> (f8E8M0FNU)-> vector<2xbf16>
llvm.return
}
+
+// CHECK-LABEL: @convert_f8x2_to_bf16x2_e4m3
+llvm.func @convert_f8x2_to_bf16x2_e4m3(%src : vector<2xi8>, %scale_factor : i16) {
+ // CHECK: %[[res1:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e4m3x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res1]], i16 32639)
+ %res1 = nvvm.convert.f8x2.to.bf16x2 %src : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ // CHECK: %[[res2:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e4m3x2.to.bf16x2.rn.relu.scale.n2.ue8m0(i16 %[[res2]], i16 32639)
+ %res2 = nvvm.convert.f8x2.to.bf16x2 %src {relu = true} : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ // CHECK: %[[res3:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e4m3x2.to.bf16x2.rn.satfinite.scale.n2.ue8m0(i16 %[[res3]], i16 32639)
+ %res3 = nvvm.convert.f8x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ // CHECK: %[[res4:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e4m3x2.to.bf16x2.rn.relu.satfinite.scale.n2.ue8m0(i16 %[[res4]], i16 32639)
+ %res4 = nvvm.convert.f8x2.to.bf16x2 %src {relu = true, sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ // CHECK: %[[res5:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e4m3x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res5]], i16 %{{.*}})
+ %res5 = nvvm.convert.f8x2.to.bf16x2 %src, %scale_factor : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f8x2_to_bf16x2_e5m2
+llvm.func @convert_f8x2_to_bf16x2_e5m2(%src : vector<2xi8>, %scale_factor : i16) {
+ // CHECK: %[[res1:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e5m2x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res1]], i16 32639)
+ %res1 = nvvm.convert.f8x2.to.bf16x2 %src : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+ // CHECK: %[[res2:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e5m2x2.to.bf16x2.rn.relu.scale.n2.ue8m0(i16 %[[res2]], i16 32639)
+ %res2 = nvvm.convert.f8x2.to.bf16x2 %src {relu = true} : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+ // CHECK: %[[res3:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e5m2x2.to.bf16x2.rn.satfinite.scale.n2.ue8m0(i16 %[[res3]], i16 32639)
+ %res3 = nvvm.convert.f8x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+ // CHECK: %[[res4:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e5m2x2.to.bf16x2.rn.relu.satfinite.scale.n2.ue8m0(i16 %[[res4]], i16 32639)
+ %res4 = nvvm.convert.f8x2.to.bf16x2 %src {relu = true, sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+ // CHECK: %[[res5:.*]] = bitcast <2 x i8> %{{.*}} to i16
+ // CHECK-NEXT: %{{.*}} = call <2 x bfloat> @llvm.nvvm.e5m2x2.to.bf16x2.rn.scale.n2.ue8m0(i16 %[[res5]], i16 %{{.*}})
+ %res5 = nvvm.convert.f8x2.to.bf16x2 %src, %scale_factor : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 6e96e918d5f0d..1f2372abd3189 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -226,7 +226,7 @@ llvm.func @nvvm_cvt_f32x2_to_f4x2_invalid_type(%a : f32, %b : f32) {
// -----
llvm.func @nvvm_cvt_f8x2_to_f16x2_invalid_type(%src : vector<2xi8>) {
- // expected-error @below {{Only 'f8E4M3FN' and 'f8E5M2' types are supported for conversions from f8x2 to f16x2.}}
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f8E4M3FN type or f8E5M2 type}}
%res = nvvm.convert.f8x2.to.f16x2 %src : vector<2xi8> (f8E4M3) -> vector<2xf16>
llvm.return
}
@@ -234,29 +234,69 @@ llvm.func @nvvm_cvt_f8x2_to_f16x2_invalid_type(%src : vector<2xi8>) {
// -----
llvm.func @nvvm_cvt_f8x2_to_bf16x2_invalid_type(%src : vector<2xi8>) {
- // expected-error @below {{Only 'f8E8M0FNU' type is supported for conversions from f8x2 to bf16x2.}}
- %res = nvvm.convert.f8x2.to.bf16x2 %src : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f8E8M0FNU type or f8E4M3FN type or f8E5M2 type}}
+ %res = nvvm.convert.f8x2.to.bf16x2 %src : vector<2xi8> (f8E4M3) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_f8x2_to_bf16x2_ue8m0_invalid_sat(%src : vector<2xi8>) {
+ // expected-error @below {{Only NONE saturation mode is supported for conversions from 'f8E8M0FNU' type}}
+ %res = nvvm.convert.f8x2.to.bf16x2 %src {sat = #nvvm.sat_mode<satfinite>} : vector<2xi8> (f8E8M0FNU) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_f8x2_to_bf16x2_ue8m0_invalid_scale(%src : vector<2xi8>, %sf : i16) {
+ // expected-error @below {{scaleFactor not supported for conversions from 'f8E8M0FNU' type}}
+ %res = nvvm.convert.f8x2.to.bf16x2 %src, %sf : vector<2xi8> (f8E8M0FNU) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_f8x2_to_bf16x2_ue8m0_invalid_relu(%src : vector<2xi8>) {
+ // expected-error @below {{relu not supported for conversions from 'f8E8M0FNU' type}}
+ %res = nvvm.convert.f8x2.to.bf16x2 %src {relu = true} : vector<2xi8> (f8E8M0FNU) -> vector<2xbf16>
llvm.return
}
// -----
llvm.func @nvvm_cvt_f6x2_to_f16x2_invalid_type(%src : vector<2xi8>) {
- // expected-error @below {{Only 'f6E2M3FN' and 'f6E3M2FN' types are supported for conversions from f6x2 to f16x2.}}
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f6E2M3FN type or f6E3M2FN type}}
%res = nvvm.convert.f6x2.to.f16x2 %src : vector<2xi8> (f8E4M3FN) -> vector<2xf16>
llvm.return
}
// -----
+llvm.func @nvvm_cvt_f6x2_to_bf16x2_invalid_type(%src : vector<2xi8>) {
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f6E2M3FN type or f6E3M2FN type}}
+ %res = nvvm.convert.f6x2.to.bf16x2 %src : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
llvm.func @nvvm_cvt_f4x2_to_f16x2_invalid_type(%src : i8) {
- // expected-error @below {{Only 'f4E2M1FN' type is supported for conversions from f4x2 to f16x2.}}
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f4E2M1FN type}}
%res = nvvm.convert.f4x2.to.f16x2 %src : i8 (f6E2M3FN) -> vector<2xf16>
llvm.return
}
// -----
+llvm.func @nvvm_cvt_f4x2_to_bf16x2_invalid_type(%src : i8) {
+ // expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f4E2M1FN type}}
+ %res = nvvm.convert.f4x2.to.bf16x2 %src : i8 (f6E2M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
llvm.func @nvvm_prefetch_L1_with_evict_priority(%global_ptr: !llvm.ptr<1>) {
// expected-error @below {{cache eviction priority supported only for cache level L2}}
nvvm.prefetch level = L1, evict_priority = evict_last, %global_ptr : !llvm.ptr<1>
>From 015714cdefde2566e89d2edd2fddaf11626bb7d5 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 29 May 2026 09:19:04 +0000
Subject: [PATCH 2/5] update description formatting
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index b77cae747af39..607e3bdf3cca4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2186,16 +2186,18 @@ class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstTy
The result `dst` is represented as a vector of }] # !tolower(dstType) # [{ elements.
- The `relu` attribute, when set, lowers to the '.relu' variant of
+ The `relu` attribute, when set, lowers to the '.relu' variant of
the cvt instruction.}] #
!if(!eq(dstType, "BF16"),
- [{The `sat` attribute specifies the saturation mode.
-
- The optional scaling-factors for each of the inputs are provided through
- the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported
+ [{
+
+ The `sat` attribute specifies the saturation mode.
+
+ The optional scaling-factors for each of the inputs are provided through
+ the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported
as the type of the scale-factor currently.}], "") # [{
-
+
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
let results = (outs VectorOfLengthAndType<[2], [!cast<Type>(dstType)]>:$dst);
>From 7bd6b0f7d733116e788efc6d313fe46627ea61ec Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 29 May 2026 09:20:58 +0000
Subject: [PATCH 3/5] fix formatting
---
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 46 +++++++++++-----------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 17027b5622d79..fa962c8b031b7 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -4792,36 +4792,36 @@ NVVM::IDArgPair ConvertF8x2ToBF16x2Op::getIntrinsicIDAndArgs(
bool hasScale = static_cast<bool>(curOp.getScaleFactor());
bool hasSatfinite = curOp.getSat() == NVVM::SaturationMode::SATFINITE;
bool hasRelu = curOp.getRelu();
-
+
static constexpr llvm::Intrinsic::ID E4M3Ids[] = {
- llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e4m3x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
};
static constexpr llvm::Intrinsic::ID E5M2Ids[] = {
- llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
- llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0,
+ llvm::Intrinsic::nvvm_e5m2x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0,
};
llvm::Intrinsic::ID intId =
- llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(curOp.getSrcType())
- .Case([&](Float8E8M0FNUType type) {
- return llvm::Intrinsic::nvvm_ue8m0x2_to_bf16x2;
- })
- .Case([&](Float8E4M3FNType type) {
- return E4M3Ids[hasSatfinite << 1 | hasRelu];
- })
- .Case([&](Float8E5M2Type type) {
- return E5M2Ids[hasSatfinite << 1 | hasRelu];
- })
- .Default([](mlir::Type type) {
- llvm_unreachable("Invalid type for ConvertF8x2ToBF16x2Op");
- return llvm::Intrinsic::not_intrinsic;
- });
+ llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(curOp.getSrcType())
+ .Case([&](Float8E8M0FNUType type) {
+ return llvm::Intrinsic::nvvm_ue8m0x2_to_bf16x2;
+ })
+ .Case([&](Float8E4M3FNType type) {
+ return E4M3Ids[hasSatfinite << 1 | hasRelu];
+ })
+ .Case([&](Float8E5M2Type type) {
+ return E5M2Ids[hasSatfinite << 1 | hasRelu];
+ })
+ .Default([](mlir::Type type) {
+ llvm_unreachable("Invalid type for ConvertF8x2ToBF16x2Op");
+ return llvm::Intrinsic::not_intrinsic;
+ });
llvm::Value *packedI16 =
builder.CreateBitCast(mt.lookupValue(curOp.getSrc()),
llvm::Type::getInt16Ty(builder.getContext()));
>From 52b3614caa7ce0b167b15f8e79dec6b8d6024dc6 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 1 Jun 2026 17:38:15 +0000
Subject: [PATCH 4/5] address comments
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 96 +++++++++++++++++----
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 20 +++--
2 files changed, 92 insertions(+), 24 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 607e3bdf3cca4..52c98f299d81c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2176,20 +2176,20 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
}];
}
-class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstType, list<Type> supportedTypes, int needVerify = 0>
-: NVVM_SingleResultIntrinsicOp<"convert." # !tolower(srcType) # "x2.to." # !tolower(dstType) # "x2", [], "$dst"> {
- let summary = "Convert a pair of " # !tolower(srcType) # " inputs to " # !tolower(dstType) # "x2";
+class NVVM_ConvertToFP16x2Op_Base <string srcTypeStr, Type srcStorageType, string dstTypeStr, list<Type> supportedTypes, int needVerify = 0>
+: NVVM_SingleResultIntrinsicOp<"convert." # !tolower(srcTypeStr) # "x2.to." # !tolower(dstTypeStr) # "x2", [], "$dst"> {
+ let summary = "Convert a pair of " # !tolower(srcTypeStr) # " inputs to " # !tolower(dstTypeStr) # "x2";
let description = [{
- This Op converts the given }] # !tolower(srcType) # [{ inputs in a }] #
- !if(!eq(srcType, "F4"), "packed i8", "i8x2 vector") # [{ to }] #
- !tolower(dstType) # [{.
+ This Op converts the given }] # !tolower(srcTypeStr) # [{ inputs in a }] #
+ !if(!eq(srcTypeStr, "F4"), "packed i8", "i8x2 vector") # [{ to }] #
+ !tolower(dstTypeStr) # [{.
- The result `dst` is represented as a vector of }] # !tolower(dstType) # [{ elements.
+ The result `dst` is represented as a vector of }] # !tolower(dstTypeStr) # [{ elements.
The `relu` attribute, when set, lowers to the '.relu' variant of
the cvt instruction.}] #
- !if(!eq(dstType, "BF16"),
+ !if(!eq(dstTypeStr, "BF16"),
[{
The `sat` attribute specifies the saturation mode.
@@ -2200,18 +2200,18 @@ class NVVM_ConvertToFP16x2Op_Base <string srcType, Type srcArgType, string dstTy
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
- let results = (outs VectorOfLengthAndType<[2], [!cast<Type>(dstType)]>:$dst);
- let arguments = !if(!eq(dstType, "F16"),
- (ins srcArgType:$src,
+ let results = (outs VectorOfLengthAndType<[2], [!cast<Type>(dstTypeStr)]>:$dst);
+ let arguments = !if(!eq(dstTypeStr, "F16"),
+ (ins srcStorageType:$src,
TypeAttrOf<AnyTypeOf<supportedTypes>>:$srcType,
DefaultValuedAttr<BoolAttr, "false">:$relu),
- (ins srcArgType:$src,
+ (ins srcStorageType:$src,
Optional<I16>:$scaleFactor,
TypeAttrOf<AnyTypeOf<supportedTypes>>:$srcType,
DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu));
let assemblyFormat =
- !if(!eq(dstType, "F16"),
+ !if(!eq(dstTypeStr, "F16"),
"$src attr-dict `:` type($src) `(` $srcType `)` `->` type($dst)",
"$src (`,` $scaleFactor^)? attr-dict `:` type($src) `(` $srcType `)` `->` type($dst)");
let hasVerifier = needVerify;
@@ -2221,18 +2221,78 @@ def NVVM_ConvertF8x2ToF16x2Op :
NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "F16",
[F8E4M3FN, F8E5M2]>;
def NVVM_ConvertF8x2ToBF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "BF16",
- [F8E8M0FNU, F8E4M3FN, F8E5M2], 1>;
+ NVVM_ConvertToFP16x2Op_Base<"F8", VectorOfLengthAndType<[2], [I8]>, "BF16",
+ [F8E8M0FNU, F8E4M3FN, F8E5M2], 1> {
+ let append description = [{
+
+ Example:
+
+ ```mlir
+ // Basic conversion from f8E4M3FN.
+ %res1 = nvvm.convert.f8x2.to.bf16x2 %src
+ : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+
+ // Conversion from f8E5M2 with relu and saturation.
+ %res2 = nvvm.convert.f8x2.to.bf16x2 %src
+ {relu = true, sat = #nvvm.sat_mode<satfinite>}
+ : vector<2xi8> (f8E5M2) -> vector<2xbf16>
+
+ // Conversion with a packed ue8m0 scale-factor.
+ %res3 = nvvm.convert.f8x2.to.bf16x2 %src, %scaleFactor
+ : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ ```
+ }];
+}
def NVVM_ConvertF6x2ToF16x2Op :
NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "F16",
[F6E2M3FN, F6E3M2FN]>;
def NVVM_ConvertF6x2ToBF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "BF16",
- [F6E2M3FN, F6E3M2FN]>;
+ NVVM_ConvertToFP16x2Op_Base<"F6", VectorOfLengthAndType<[2], [I8]>, "BF16",
+ [F6E2M3FN, F6E3M2FN]> {
+ let append description = [{
+
+ Example:
+
+ ```mlir
+ // Basic conversion from f6E2M3FN.
+ %res1 = nvvm.convert.f6x2.to.bf16x2 %src
+ : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+
+ // Conversion from f6E3M2FN with relu and saturation.
+ %res2 = nvvm.convert.f6x2.to.bf16x2 %src
+ {relu = true, sat = #nvvm.sat_mode<satfinite>}
+ : vector<2xi8> (f6E3M2FN) -> vector<2xbf16>
+
+ // Conversion with a packed ue8m0 scale-factor.
+ %res3 = nvvm.convert.f6x2.to.bf16x2 %src, %scaleFactor
+ : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ ```
+ }];
+}
def NVVM_ConvertF4x2ToF16x2Op :
NVVM_ConvertToFP16x2Op_Base<"F4", I8, "F16", [F4E2M1FN]>;
def NVVM_ConvertF4x2ToBF16x2Op :
- NVVM_ConvertToFP16x2Op_Base<"F4", I8, "BF16", [F4E2M1FN]>;
+ NVVM_ConvertToFP16x2Op_Base<"F4", I8, "BF16", [F4E2M1FN]> {
+ let append description = [{
+
+ Example:
+
+ ```mlir
+ // Basic conversion; the f4x2 source is packed in a single i8.
+ %res1 = nvvm.convert.f4x2.to.bf16x2 %src
+ : i8 (f4E2M1FN) -> vector<2xbf16>
+
+ // Conversion with relu and saturation.
+ %res2 = nvvm.convert.f4x2.to.bf16x2 %src
+ {relu = true, sat = #nvvm.sat_mode<satfinite>}
+ : i8 (f4E2M1FN) -> vector<2xbf16>
+
+ // Conversion with a packed ue8m0 scale-factor.
+ %res3 = nvvm.convert.f4x2.to.bf16x2 %src, %scaleFactor
+ : i8 (f4E2M1FN) -> vector<2xbf16>
+ ```
+ }];
+}
def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> {
let summary = "Convert a pair of f32 inputs to S2F6x2";
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index fa962c8b031b7..8aac5a672fbf0 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -4829,8 +4829,10 @@ NVVM::IDArgPair ConvertF8x2ToBF16x2Op::getIntrinsicIDAndArgs(
llvm::SmallVector<llvm::Value *> args;
args.push_back(packedI16);
if (!isa<Float8E8M0FNUType>(curOp.getSrcType()))
- args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
- : builder.getInt16(0x7f7f));
+ args.push_back(
+ hasScale ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(0x7f7f)); // default scale factor (value of
+ // 1 for both elements)
return {intId, std::move(args)};
}
@@ -4900,8 +4902,11 @@ NVVM::IDArgPair ConvertF6x2ToBF16x2Op::getIntrinsicIDAndArgs(
llvm::SmallVector<llvm::Value *> args;
args.push_back(packedI16);
- args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
- : builder.getInt16(0x7f7f));
+ args.push_back(
+ hasScale
+ ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(
+ 0x7f7f)); // default scale factor (value of 1 for both elements)
return {intId, std::move(args)};
}
@@ -4959,8 +4964,11 @@ NVVM::IDArgPair ConvertF4x2ToBF16x2Op::getIntrinsicIDAndArgs(
llvm::SmallVector<llvm::Value *> args;
args.push_back(extendedI16);
- args.push_back(hasScale ? mt.lookupValue(curOp.getScaleFactor())
- : builder.getInt16(0x7f7f));
+ args.push_back(
+ hasScale
+ ? mt.lookupValue(curOp.getScaleFactor())
+ : builder.getInt16(
+ 0x7f7f)); // default scale factor (value of 1 for both elements)
return {intId, std::move(args)};
}
>From f34807356880ccf6e7699a386decdad537aa304a Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 8 Jun 2026 10:32:18 +0000
Subject: [PATCH 5/5] add saturation mode constraint
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 8 ++++--
mlir/test/Target/LLVMIR/nvvmir-invalid.mlir | 32 +++++++++++++++++++++
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 52c98f299d81c..f0c404ba57ebe 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2176,6 +2176,10 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
}];
}
+def SaturationModeSatfiniteOrNone :
+ ConfinedAttr<SaturationModeAttr, [EnumAttrIsOneOf<SaturationModeAttr,
+ [SaturationModeNone, SaturationModeFinite]>]>;
+
class NVVM_ConvertToFP16x2Op_Base <string srcTypeStr, Type srcStorageType, string dstTypeStr, list<Type> supportedTypes, int needVerify = 0>
: NVVM_SingleResultIntrinsicOp<"convert." # !tolower(srcTypeStr) # "x2.to." # !tolower(dstTypeStr) # "x2", [], "$dst"> {
let summary = "Convert a pair of " # !tolower(srcTypeStr) # " inputs to " # !tolower(dstTypeStr) # "x2";
@@ -2208,7 +2212,7 @@ class NVVM_ConvertToFP16x2Op_Base <string srcTypeStr, Type srcStorageType, strin
(ins srcStorageType:$src,
Optional<I16>:$scaleFactor,
TypeAttrOf<AnyTypeOf<supportedTypes>>:$srcType,
- DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
+ DefaultValuedAttr<SaturationModeSatfiniteOrNone, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu));
let assemblyFormat =
!if(!eq(dstTypeStr, "F16"),
@@ -2399,7 +2403,7 @@ def NVVM_ConvertS2F6x2ToBF16x2Op : NVVM_SingleResultIntrinsicOp<"convert.s2f6x2.
let results = (outs VectorOfLengthAndType<[2], [BF16]>:$dst);
let arguments = (ins VectorOfLengthAndType<[2], [I8]>:$src,
Optional<I16>:$scaleFactor,
- DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
+ DefaultValuedAttr<SaturationModeSatfiniteOrNone, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu);
let assemblyFormat =
"$src (`,` $scaleFactor^)? attr-dict `:` type($src) `->` type($dst)";
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 1f2372abd3189..d215164b28105 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -265,6 +265,38 @@ llvm.func @nvvm_cvt_f8x2_to_bf16x2_ue8m0_invalid_relu(%src : vector<2xi8>) {
// -----
+llvm.func @nvvm_cvt_f8x2_to_bf16x2_invalid_sat(%src : vector<2xi8>) {
+ // expected-error @below {{op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %res = nvvm.convert.f8x2.to.bf16x2 %src {sat = #nvvm.sat_mode<sat>} : vector<2xi8> (f8E4M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_f6x2_to_bf16x2_invalid_sat(%src : vector<2xi8>) {
+ // expected-error @below {{op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %res = nvvm.convert.f6x2.to.bf16x2 %src {sat = #nvvm.sat_mode<sat>} : vector<2xi8> (f6E2M3FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_f4x2_to_bf16x2_invalid_sat(%src : i8) {
+ // expected-error @below {{op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %res = nvvm.convert.f4x2.to.bf16x2 %src {sat = #nvvm.sat_mode<sat>} : i8 (f4E2M1FN) -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
+llvm.func @nvvm_cvt_s2f6x2_to_bf16x2_invalid_sat(%src : vector<2xi8>) {
+ // expected-error @below {{op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %res = nvvm.convert.s2f6x2.to.bf16x2 %src {sat = #nvvm.sat_mode<sat>} : vector<2xi8> -> vector<2xbf16>
+ llvm.return
+}
+
+// -----
+
llvm.func @nvvm_cvt_f6x2_to_f16x2_invalid_type(%src : vector<2xi8>) {
// expected-error @below {{op attribute 'srcType' failed to satisfy constraint: type attribute of f6E2M3FN type or f6E3M2FN type}}
%res = nvvm.convert.f6x2.to.f16x2 %src : vector<2xi8> (f8E4M3FN) -> vector<2xf16>
More information about the Mlir-commits
mailing list