[Mlir-commits] [mlir] [MLIR][NVVM] Add support for pzo in FP8/FP6/FP4 convert Ops (PR #229660)
Dharuni R Acharya
llvmlistbot at llvm.org
Thu Oct 8 01:16:04 PDT 2026
https://github.com/DharuniRAcharya updated https://github.com/llvm/llvm-project/pull/229660
>From 85ace0dee4a2a622467f47bfc8f4ad6ee79670b0 Mon Sep 17 00:00:00 2001
From: DharuniRAcharya <dharunira at nvidia.com>
Date: Wed, 7 Oct 2026 05:21:39 +0000
Subject: [PATCH 1/2] [MLIR][NVVM] Add support for pzo in FP8/FP6/FP4 convert
Ops
This patch adds support for pzo and rz rounding mode to existing FP8, FP6 and FP4 Convert Ops.
Lit tests are added to verify the lowering to the intrinsics.
Negative tests are also added to check the error-handling of invalid combinations.
PTX ISA Reference: https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt
Signed-off-by: DharuniRAcharya <dharunira at nvidia.com>
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 112 +++++--
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 288 +++++++++++-------
.../invalid-convert-stochastic-rounding.mlir | 2 +-
.../Target/LLVMIR/nvvm/convert_fp4x2.mlir | 69 +++++
.../LLVMIR/nvvm/convert_fp4x2_invalid.mlir | 16 +
.../Target/LLVMIR/nvvm/convert_fp6x2.mlir | 66 ++++
.../LLVMIR/nvvm/convert_fp6x2_invalid.mlir | 16 +
.../Target/LLVMIR/nvvm/convert_fp8x2.mlir | 90 ++++++
.../LLVMIR/nvvm/convert_fp8x2_invalid.mlir | 10 +-
mlir/test/Target/LLVMIR/nvvmir-invalid.mlir | 14 +-
10 files changed, 541 insertions(+), 142 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index b15ee81291e5ca..79eda3fee335cc 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2046,8 +2046,10 @@ def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> {
packed such that the value converted from `a` is stored in the upper 4 bits
of `dst` and the value converted from `b` is stored in the lower 4 bits of
`dst`.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2055,10 +2057,15 @@ def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> {
let results = (outs I8:$dst);
let arguments = (ins F32:$a, F32:$b,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a `,` $b oilist(`relu` `=` $relu) attr-dict
- `:` type($dst) `(` $dstTy `)`
+ $a `,` $b oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo)
+ attr-dict `:` type($dst) `(` $dstTy `)`
}];
let hasVerifier = 1;
@@ -2085,17 +2092,24 @@ class NVVM_ConvertFPx2ToF4x2Op<string srcType>
packed such that the value converted from the first element of `a` is
stored in the lower 4 bits of `dst` and the value converted from the second
element of `a` is stored in the upper 4 bits of `dst`.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
}];
let results = (outs I8:$dst);
let arguments = (ins
VectorOfLengthAndType<[2], [!cast<Type>(srcType)]>:$src,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<F4E2M1FN>:$dstTy);
+ TypeAttrOf<F4E2M1FN>:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $src oilist(`relu` `=` $relu) attr-dict
+ $src oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
@@ -2127,8 +2141,10 @@ def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> {
stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2138,22 +2154,28 @@ def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> {
F32:$a,
F32:$b,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a `,` $b oilist(`relu` `=` $relu) attr-dict
- `:` type($dst) `(` $dstTy `)`
+ $a `,` $b oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo)
+ attr-dict `:` type($dst) `(` $dstTy `)`
}];
let hasVerifier = 1;
let extraClassDeclaration = [{
static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::ConvertF32x2ToF6x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::ConvertF32x2ToF6x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$a, $b, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$a, $b, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2176,27 +2198,36 @@ class NVVM_ConvertFPx2ToF6x2Op<string srcType>
`dst` with 2 MSB bits padded with zeros.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector with 2 MSB bits padded with zeros.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
}];
let results = (outs AnyTypeOf<[I16, VectorOfLengthAndType<[2], [I8]>]>:$dst);
let arguments = (ins
VectorOfLengthAndType<[2], [!cast<Type>(srcType)]>:$src,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<AnyTypeOf<[F6E2M3FN, F6E3M2FN]>>:$dstTy);
+ TypeAttrOf<AnyTypeOf<[F6E2M3FN, F6E3M2FN]>>:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $src oilist(`relu` `=` $relu) attr-dict
+ $src oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
let extraClassDeclaration = [{
- static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy, bool hasRelu);
+ static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
+ bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::Convert}]#srcType#[{x2ToF6x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::Convert}]#srcType#[{x2ToF6x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$src, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$src, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2220,8 +2251,11 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
The `rnd` and `sat` attributes specify the rounding and saturation modes respectively.
+ Conversions to `f8E4M3FN` and `f8E5M2` support `rn` and `rz` rounding.
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction. `pzo` is not supported for
+ conversions to `f8E8M0FNU`.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2234,10 +2268,11 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
DefaultValuedAttr<FPRoundingModeAttr, "FPRoundingMode::NONE">:$rnd,
DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
$a `,` $b oilist(
- `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu
+ `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu | `pzo` `=` $pzo
) attr-dict `:` type($dst) `(` $dstTy `)`
}];
@@ -2252,7 +2287,7 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
auto intId = NVVM::ConvertF32x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $sat, $relu);
llvm::SmallVector<llvm::Value *> args = {$a, $b};
if (!mlir::isa<mlir::Float8E8M0FNUType>($dstTy))
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1($pzo));
llvm::Value *packedI16 = createIntrinsicCall(builder, intId, args);
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
@@ -2275,8 +2310,10 @@ def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> {
second element of `a` is stored in the lower 8 bits of `dst`.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2286,21 +2323,27 @@ def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> {
let arguments = (ins
VectorOfLengthAndType<[2], [F16]>:$a,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a oilist(`relu` `=` $relu) attr-dict
+ $a oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($a) `->` type($dst) `(` $dstTy `)`
}];
let extraClassDeclaration = [{
static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::ConvertF16x2ToF8x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::ConvertF16x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$a, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$a, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2322,7 +2365,11 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
The `rnd` and `sat` attributes specify the rounding and saturation modes
- respectively.
+ respectively. Conversions to `f8E4M3FN` and `f8E5M2` support `rn` and `rz`
+ rounding. The `relu` attribute, when set, lowers to the '.relu' variant of
+ the cvt instruction. The `pzo` attribute, when set, lowers to the '.pzo'
+ variant of the cvt instruction. `pzo` is not supported for conversions to
+ `f8E8M0FNU`.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2334,10 +2381,11 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
DefaultValuedAttr<FPRoundingModeAttr, "FPRoundingMode::NONE">:$rnd,
DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<AnyTypeOf<[F8E8M0FNU, F8E4M3FN, F8E5M2]>>:$dstTy);
+ TypeAttrOf<AnyTypeOf<[F8E8M0FNU, F8E4M3FN, F8E5M2]>>:$dstTy,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
$src oilist(
- `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu
+ `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu | `pzo` `=` $pzo
) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
@@ -2352,7 +2400,7 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
auto intId = NVVM::ConvertBF16x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $sat, $relu);
llvm::SmallVector<llvm::Value *> args = {$src};
if (!mlir::isa<mlir::Float8E8M0FNUType>($dstTy))
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1($pzo));
llvm::Value *packedI16 = createIntrinsicCall(builder, intId, args);
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 21c7c1cc148798..1e758f34b123aa 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -548,15 +548,16 @@ LogicalResult ConvertF32x2ToF8x2Op::verify() {
bool isSatFinite = getSat() == SatMode::SATFINITE;
bool hasRelu = getRelu();
+ bool hasPzo = getPzo();
mlir::MLIRContext *ctx = getContext();
return llvm::TypeSwitch<mlir::Type, LogicalResult>(getDstTy())
.Case<mlir::Float8E4M3FNType, mlir::Float8E5M2Type>(
[&](mlir::Type) -> LogicalResult {
- if (!isRoundingModeRN) {
- return emitOpError("Only RN rounding mode is supported for "
- "conversions from f32x2 to ")
+ if (!(isRoundingModeRN || isRoundingModeRZ)) {
+ return emitOpError("Only RN and RZ rounding modes are supported "
+ "for conversions from f32x2 to ")
<< mlir::Float8E4M3FNType::get(ctx) << " and "
<< mlir::Float8E5M2Type::get(ctx) << " types";
}
@@ -579,6 +580,10 @@ LogicalResult ConvertF32x2ToF8x2Op::verify() {
return emitOpError("relu not supported for conversions to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
}
+ if (hasPzo) {
+ return emitOpError("pzo not supported for conversions to ")
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
+ }
return success();
})
.Default([&](mlir::Type) {
@@ -612,15 +617,16 @@ LogicalResult ConvertBF16x2ToF8x2Op::verify() {
bool isRoundingModeRP = getRnd() == RndMode::RP;
bool isSatFinite = getSat() == SatMode::SATFINITE;
bool hasRelu = getRelu();
+ bool hasPzo = getPzo();
mlir::MLIRContext *ctx = getContext();
return llvm::TypeSwitch<mlir::Type, LogicalResult>(getDstTy())
.Case<mlir::Float8E4M3FNType, mlir::Float8E5M2Type>(
[&](mlir::Type) -> LogicalResult {
- if (!isRoundingModeRN)
- return emitOpError("Only RN rounding mode is supported for "
- "conversions from bf16x2 to ")
+ if (!(isRoundingModeRN || isRoundingModeRZ))
+ return emitOpError("Only RN and RZ rounding modes are supported "
+ "for conversions from bf16x2 to ")
<< mlir::Float8E4M3FNType::get(ctx) << " and "
<< mlir::Float8E5M2Type::get(ctx) << " types";
if (!isSatFinite)
@@ -638,6 +644,9 @@ LogicalResult ConvertBF16x2ToF8x2Op::verify() {
if (hasRelu)
return emitOpError("relu not supported for conversions to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
+ if (hasPzo)
+ return emitOpError("pzo not supported for conversions to ")
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
return success();
})
.Default([&](mlir::Type) -> LogicalResult {
@@ -5086,30 +5095,42 @@ ConvertF32x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertF32x2ToF4x2Op op,
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getA()));
args.push_back(mt.lookupValue(op.getB()));
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1(op.getPzo()));
- bool hasRelu = op.getRelu();
-
- llvm::Intrinsic::ID intId =
- hasRelu ? llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_satfinite;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rz_relu_satfinite,
+ };
- return {intId, std::move(args)};
+ return {ids[idx], std::move(args)};
}
-#define GET_F32x2_TO_F6x2_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_ff_to_##type##_rn_relu_satfinite \
- : llvm::Intrinsic::nvvm_ff_to_##type##_rn_satfinite
+llvm::Intrinsic::ID
+ConvertF32x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rz_relu_satfinite,
+ };
-llvm::Intrinsic::ID ConvertF32x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float6E2M3FNType) {
- return GET_F32x2_TO_F6x2_ID(e2m3x2, hasRelu);
- })
- .Case([&](mlir::Float6E3M2FNType) {
- return GET_F32x2_TO_F6x2_ID(e3m2x2, hasRelu);
- })
+ .Case([&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case([&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF32x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
@@ -5121,17 +5142,24 @@ ConvertF16x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertF16x2ToF4x2Op &op,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
mlir::Type dstTy = op.getDstTy();
- bool hasRelu = op.getRelu();
- llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rz_relu_satfinite,
+ };
+ llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
if (llvm::isa<mlir::Float4E2M1FNType>(dstTy))
- intId = hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_satfinite;
+ intId = ids[idx];
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getSrc()));
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1(op.getPzo()));
return {intId, std::move(args)};
}
@@ -5141,86 +5169,121 @@ ConvertBF16x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertBF16x2ToF4x2Op &op,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
mlir::Type dstTy = op.getDstTy();
- bool hasRelu = op.getRelu();
- llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rz_relu_satfinite,
+ };
+ llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
if (llvm::isa<mlir::Float4E2M1FNType>(dstTy))
- intId = hasRelu ? llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_satfinite;
+ intId = ids[idx];
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getSrc()));
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1(op.getPzo()));
return {intId, std::move(args)};
}
-llvm::Intrinsic::ID ConvertF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
+llvm::Intrinsic::ID
+ConvertF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rz_relu_satfinite,
+ };
+
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float6E2M3FNType>([&](mlir::Float6E2M3FNType) {
- return hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_satfinite;
- })
- .Case<mlir::Float6E3M2FNType>([&](mlir::Float6E3M2FNType) {
- return hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_satfinite;
- })
+ .Case<mlir::Float6E2M3FNType>(
+ [&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case<mlir::Float6E3M2FNType>(
+ [&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF16x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
});
}
-llvm::Intrinsic::ID ConvertBF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
+llvm::Intrinsic::ID
+ConvertBF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd, bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rz_relu_satfinite,
+ };
+
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float6E2M3FNType>([&](mlir::Float6E2M3FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_satfinite;
- })
- .Case<mlir::Float6E3M2FNType>([&](mlir::Float6E3M2FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_satfinite;
- })
+ .Case<mlir::Float6E2M3FNType>(
+ [&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case<mlir::Float6E3M2FNType>(
+ [&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertBF16x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
});
}
-#define GET_F32x2_TO_F8X2_US_ID(rnd, has_satf) \
- has_satf ? llvm::Intrinsic::nvvm_ff_to_ue8m0x2_##rnd##_satfinite \
- : llvm::Intrinsic::nvvm_ff_to_ue8m0x2_##rnd
-
-#define GET_F32x2_TO_F8X2_S_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_ff_to_##type##_rn_relu \
- : llvm::Intrinsic::nvvm_ff_to_##type##_rn
-
llvm::Intrinsic::ID
ConvertF32x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
NVVM::SaturationMode sat, bool hasRelu) {
bool hasSatFinite = (sat == NVVM::SaturationMode::SATFINITE);
- bool hasRoundingModeRZ = (rnd == NVVM::FPRoundingMode::RZ);
- bool hasRoundingModeRP = (rnd == NVVM::FPRoundingMode::RP);
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ unsigned ue8m0Idx = (static_cast<unsigned>(hasSatFinite) << 1) |
+ static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RP);
+
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rn,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rn_relu,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rn,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rn_relu,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID ue8m0x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rp,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rp_satfinite,
+ };
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float8E4M3FNType) {
- return GET_F32x2_TO_F8X2_S_ID(e4m3x2, hasRelu);
- })
- .Case([&](mlir::Float8E5M2Type) {
- return GET_F32x2_TO_F8X2_S_ID(e5m2x2, hasRelu);
- })
+ .Case([&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case([&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
.Case([&](mlir::Float8E8M0FNUType) {
- if (hasRoundingModeRZ)
- return GET_F32x2_TO_F8X2_US_ID(rz, hasSatFinite);
- else if (hasRoundingModeRP)
- return GET_F32x2_TO_F8X2_US_ID(rp, hasSatFinite);
-
- llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
+ if (rnd != NVVM::FPRoundingMode::RZ && rnd != NVVM::FPRoundingMode::RP)
+ llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
+ return ue8m0x2IDs[ue8m0Idx];
})
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
@@ -5228,19 +5291,27 @@ ConvertF32x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
});
}
-#define GET_F16x2_TO_F8X2_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_f16x2_to_##type##_rn_relu \
- : llvm::Intrinsic::nvvm_f16x2_to_##type##_rn
+llvm::Intrinsic::ID
+ConvertF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rn,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rn_relu,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rz,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rn,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rn_relu,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rz,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rz_relu,
+ };
-llvm::Intrinsic::ID ConvertF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float8E4M3FNType) {
- return GET_F16x2_TO_F8X2_ID(e4m3x2, hasRelu);
- })
- .Case([&](mlir::Float8E5M2Type) {
- return GET_F16x2_TO_F8X2_ID(e5m2x2, hasRelu);
- })
+ .Case([&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case([&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF16x2ToF8x2Op");
return llvm::Intrinsic::not_intrinsic;
@@ -5252,7 +5323,23 @@ ConvertBF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
NVVM::FPRoundingMode rnd,
NVVM::SaturationMode sat, bool hasRelu) {
bool hasSatFinite = (sat == NVVM::SaturationMode::SATFINITE);
-
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ unsigned ue8m0Idx = (static_cast<unsigned>(hasSatFinite) << 1) |
+ static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RP);
+
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rz_relu_satfinite,
+ };
static constexpr llvm::Intrinsic::ID ue8m0x2IDs[] = {
llvm::Intrinsic::nvvm_bf16x2_to_ue8m0x2_rz,
llvm::Intrinsic::nvvm_bf16x2_to_ue8m0x2_rp,
@@ -5261,21 +5348,12 @@ ConvertBF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
};
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float8E4M3FNType>([&](mlir::Float8E4M3FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_satfinite;
- })
- .Case<mlir::Float8E5M2Type>([&](mlir::Float8E5M2Type) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_satfinite;
- })
- .Case<mlir::Float8E8M0FNUType>([&](mlir::Float8E8M0FNUType) {
- bool hasRoundingModeRP = (rnd == NVVM::FPRoundingMode::RP);
- unsigned index = (hasSatFinite << 1) | hasRoundingModeRP;
- return ue8m0x2IDs[index];
- })
+ .Case<mlir::Float8E4M3FNType>(
+ [&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case<mlir::Float8E5M2Type>(
+ [&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
+ .Case<mlir::Float8E8M0FNUType>(
+ [&](mlir::Float8E8M0FNUType) { return ue8m0x2IDs[ue8m0Idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertBF16x2ToF8x2Op");
return llvm::Intrinsic::not_intrinsic;
diff --git a/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir b/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
index 608622224e8cc3..d9d293a96d6052 100644
--- a/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
+++ b/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
@@ -56,7 +56,7 @@ llvm.func @convert_float_to_tf32_rs_not_supported(%src : f32) -> i32 {
// -----
llvm.func @convert_f32x2_to_f8x2_rs_not_supported(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rs> sat = <satfinite> : i16 (f8E4M3FN)
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
index 9538b7d798f7ae..4a0401960298a1 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
@@ -37,6 +37,75 @@ llvm.func @convert_bf16x2_to_f4x2(%srcA : vector<2xbf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_f4x2_rz_pzo
+llvm.func @convert_f32x2_to_f4x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> : i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> relu = true : i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rn.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rn.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB relu = true pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i8 (f4E2M1FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f16x2_to_f4x2_rz_pzo
+llvm.func @convert_f16x2_to_f4x2_rz_pzo(%srcA : vector<2xf16>) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> relu = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rn.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.f16x2.to.f4x2 %srcA pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rn.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.f16x2.to.f4x2 %srcA relu = true pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_bf16x2_to_f4x2_rz_pzo
+llvm.func @convert_bf16x2_to_f4x2_rz_pzo(%srcA : vector<2xbf16>) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> relu = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.bf16x2.to.f4x2 %srcA pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.bf16x2.to.f4x2 %srcA relu = true pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f4x2_to_f16x2
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
index d179431fe93699..88b7a46d387f95 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
@@ -15,3 +15,19 @@ llvm.func @convert_bf16x2_to_f4x2_invalid_type(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f4x2 %src : vector<2xbf16> -> i8 (f8E4M3FN)
llvm.return
}
+
+// -----
+
+llvm.func @convert_f32x2_to_f4x2_invalid_rounding(%a : f32, %b : f32) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f32x2.to.f4x2 %a, %b rnd = <rp> : i8 (f4E2M1FN)
+ llvm.return
+}
+
+// -----
+
+llvm.func @convert_bf16x2_to_f4x2_invalid_rounding(%src : vector<2xbf16>) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.bf16x2.to.f4x2 %src rnd = <rs> : vector<2xbf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
index 21f23521405b1d..cd6cd27471ded7 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
@@ -91,6 +91,72 @@ llvm.func @convert_bf16x2_to_fp6x2_vector(%srcA : vector<2xbf16>, %scale_factor
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_fp6x2_rz_pzo
+llvm.func @convert_f32x2_to_fp6x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB relu = true pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e3m2x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> : i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e3m2x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res8 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i16 (f6E3M2FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f16x2_to_fp6x2_rz_pzo
+llvm.func @convert_f16x2_to_fp6x2_rz_pzo(%srcA : vector<2xf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rn.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f16x2.to.f6x2 %srcA pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rn.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f16x2.to.f6x2 %srcA relu = true pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e3m2x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> : vector<2xf16> -> i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e3m2x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res8 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_bf16x2_to_fp6x2_rz_pzo
+llvm.func @convert_bf16x2_to_fp6x2_rz_pzo(%srcA : vector<2xbf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.bf16x2.to.f6x2 %srcA pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.bf16x2.to.f6x2 %srcA relu = true pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e3m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> : vector<2xbf16> -> i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e3m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res8 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f6x2_to_f16x2_e2m3
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
index e993868cf1c9f8..52904807a484be 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
@@ -15,3 +15,19 @@ llvm.func @convert_bf16x2_to_f6x2_invalid_type(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f6x2 %src : vector<2xbf16> -> vector<2xi8> (f8E4M3FN)
llvm.return
}
+
+// -----
+
+llvm.func @convert_f32x2_to_f6x2_invalid_rounding(%a : f32, %b : f32) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f32x2.to.f6x2 %a, %b rnd = <rp> : i16 (f6E2M3FN)
+ llvm.return
+}
+
+// -----
+
+llvm.func @convert_f16x2_to_f6x2_invalid_rounding(%src : vector<2xf16>) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f16x2.to.f6x2 %src rnd = <rm> : vector<2xf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
index 058e28be90283c..750b81c4843e3f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
@@ -44,6 +44,36 @@ llvm.func @convert_f32x2_to_f8x2_vector_return(%srcA : f32, %srcB : f32) {
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_f8x2_rz_pzo
+llvm.func @convert_f32x2_to_f8x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz(float %{{.*}}, float %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float %{{.*}}, float %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> relu = true pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz(float %{{.*}}, float %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true pzo = true : i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz(float %{{.*}}, float %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 false)
+ %res8 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float %{{.*}}, float %{{.*}}, i1 true)
+ %res9 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res10 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> relu = true pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz(float %{{.*}}, float %{{.*}}, i1 true)
+ %res11 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res12 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true pzo = true : i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f16x2_to_f8x2_e4m3
@@ -75,6 +105,36 @@ llvm.func @convert_f16x2_to_f8x2_vector_return(%src : vector<2xf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_f16x2_to_f8x2_rz_pzo
+llvm.func @convert_f16x2_to_f8x2_rz_pzo(%src : vector<2xf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz(<2 x half> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz.relu(<2 x half> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f16x2.to.f8x2 %src pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f16x2.to.f8x2 %src relu = true pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz(<2 x half> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz.relu(<2 x half> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz(<2 x half> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz.relu(<2 x half> %{{.*}}, i1 false)
+ %res8 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> %{{.*}}, i1 true)
+ %res9 = nvvm.convert.f16x2.to.f8x2 %src pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> %{{.*}}, i1 true)
+ %res10 = nvvm.convert.f16x2.to.f8x2 %src relu = true pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz(<2 x half> %{{.*}}, i1 true)
+ %res11 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz.relu(<2 x half> %{{.*}}, i1 true)
+ %res12 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_bf16x2_to_f8x2_ue8m0
@@ -126,6 +186,36 @@ llvm.func @convert_bf16x2_to_f8x2_vector_return(%src : vector<2xbf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_bf16x2_to_f8x2_rz_pzo
+llvm.func @convert_bf16x2_to_f8x2_rz_pzo(%src : vector<2xbf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res8 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res9 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res10 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res11 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res12 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f8x2_to_f16x2
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
index 02a7600ec8304b..b12a0a586dc074 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
@@ -11,7 +11,7 @@ llvm.func @convert_bf16x2_to_f8x2_invalid_type(%src : vector<2xbf16>) {
// -----
llvm.func @convert_bf16x2_to_f8x2_invalid_rounding_1(%src : vector<2xbf16>) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from bf16x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from bf16x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rm> : vector<2xbf16> -> vector<2xi8> (f8E4M3FN)
llvm.return
}
@@ -39,3 +39,11 @@ llvm.func @convert_bf16x2_to_f8x2_invalid_relu(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rp> relu = true : vector<2xbf16> -> vector<2xi8> (f8E8M0FNU)
llvm.return
}
+
+// -----
+
+llvm.func @convert_bf16x2_to_f8x2_invalid_pzo(%src : vector<2xbf16>) {
+ // expected-error @below {{pzo not supported for conversions to 'f8E8M0FNU' type}}
+ %res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xbf16> -> vector<2xi8> (f8E8M0FNU)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 9027f760d3483e..30e3ada49c6b13 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -141,15 +141,15 @@ llvm.func @nvvm_match_sync_any(%val32: i32, %thread_mask: i32) {
// -----
llvm.func @nvvm_cvt_float_to_f8x2_invalid_rounding_e4m3(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
- %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rz> sat = <satfinite> : i16 (f8E4M3FN)
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rm> sat = <satfinite> : i16 (f8E4M3FN)
llvm.return
}
// -----
llvm.func @nvvm_cvt_float_to_f8x2_invalid_rounding_e5m2(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rp> sat = <satfinite> : i16 (f8E5M2)
llvm.return
}
@@ -188,6 +188,14 @@ llvm.func @nvvm_cvt_float_to_f8x2_relu_not_supported_ue8m0(%a : f32, %b : f32) {
// -----
+llvm.func @nvvm_cvt_float_to_f8x2_pzo_not_supported_ue8m0(%a : f32, %b : f32) {
+ // expected-error @below {{pzo not supported for conversions to 'f8E8M0FNU' type}}
+ %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rz> pzo = true : i16 (f8E8M0FNU)
+ llvm.return
+}
+
+// -----
+
llvm.func @nvvm_cvt_f16x2_to_f8x2_invalid_type(%src : vector<2xf16>) {
// expected-error @below {{Only 'f8E4M3FN' and 'f8E5M2' types are supported for conversions from f16x2 to f8x2.}}
%res = nvvm.convert.f16x2.to.f8x2 %src : vector<2xf16> -> i16 (f8E8M0FNU)
>From 7eb831ffcd7b5984e1cad5f9ade97bfbc7708dae Mon Sep 17 00:00:00 2001
From: DharuniRAcharya <dharunira at nvidia.com>
Date: Thu, 8 Oct 2026 08:15:40 +0000
Subject: [PATCH 2/2] Address comments
---
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 21 ++++++++-------------
1 file changed, 8 insertions(+), 13 deletions(-)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 1e758f34b123aa..1a300b1e7f41c7 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -576,13 +576,10 @@ LogicalResult ConvertF32x2ToF8x2Op::verify() {
"conversions from f32x2 to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
}
- if (hasRelu) {
- return emitOpError("relu not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
- }
- if (hasPzo) {
- return emitOpError("pzo not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
+ if (hasRelu || hasPzo) {
+ return emitOpError() << (hasRelu ? "relu" : "pzo")
+ << " not supported for conversions to "
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
}
return success();
})
@@ -641,12 +638,10 @@ LogicalResult ConvertBF16x2ToF8x2Op::verify() {
return emitOpError("Only RZ and RP rounding modes are supported for "
"conversions from bf16x2 to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
- if (hasRelu)
- return emitOpError("relu not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
- if (hasPzo)
- return emitOpError("pzo not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
+ if (hasRelu || hasPzo)
+ return emitOpError() << (hasRelu ? "relu" : "pzo")
+ << " not supported for conversions to "
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
return success();
})
.Default([&](mlir::Type) -> LogicalResult {
More information about the Mlir-commits
mailing list