[Mlir-commits] [mlir] 714e62b - [MLIR][NVVM] Add support for pzo in FP8/FP6/FP4 convert Ops (#229660)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Thu Oct 15 02:04:18 PDT 2026
Author: Dharuni R Acharya
Date: 2026-10-15T14:34:12+05:30
New Revision: 714e62b6d430dbbfe6e2b372c07b27d07ce2f46d
URL: https://github.com/llvm/llvm-project/commit/714e62b6d430dbbfe6e2b372c07b27d07ce2f46d
DIFF: https://github.com/llvm/llvm-project/commit/714e62b6d430dbbfe6e2b372c07b27d07ce2f46d.diff
LOG: [MLIR][NVVM] Add support for pzo in FP8/FP6/FP4 convert Ops (#229660)
This patch adds support for `pzo` and `rz` rounding mode to existing
`FP8`, `FP6` and `FP4` Convert Ops.
Lit tests are added to verify the lowering to the intrinsics.
Negative tests are also added to check the error-handling of invalid
combinations.
PTX ISA Reference:
https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt
---------
Signed-off-by: DharuniRAcharya <dharunira at nvidia.com>
Added:
Modified:
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index b15ee81291e5ca..79eda3fee335cc 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -2046,8 +2046,10 @@ def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> {
packed such that the value converted from `a` is stored in the upper 4 bits
of `dst` and the value converted from `b` is stored in the lower 4 bits of
`dst`.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2055,10 +2057,15 @@ def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> {
let results = (outs I8:$dst);
let arguments = (ins F32:$a, F32:$b,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a `,` $b oilist(`relu` `=` $relu) attr-dict
- `:` type($dst) `(` $dstTy `)`
+ $a `,` $b oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo)
+ attr-dict `:` type($dst) `(` $dstTy `)`
}];
let hasVerifier = 1;
@@ -2085,17 +2092,24 @@ class NVVM_ConvertFPx2ToF4x2Op<string srcType>
packed such that the value converted from the first element of `a` is
stored in the lower 4 bits of `dst` and the value converted from the second
element of `a` is stored in the upper 4 bits of `dst`.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
}];
let results = (outs I8:$dst);
let arguments = (ins
VectorOfLengthAndType<[2], [!cast<Type>(srcType)]>:$src,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<F4E2M1FN>:$dstTy);
+ TypeAttrOf<F4E2M1FN>:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $src oilist(`relu` `=` $relu) attr-dict
+ $src oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
@@ -2127,8 +2141,10 @@ def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> {
stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2138,22 +2154,28 @@ def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> {
F32:$a,
F32:$b,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a `,` $b oilist(`relu` `=` $relu) attr-dict
- `:` type($dst) `(` $dstTy `)`
+ $a `,` $b oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo)
+ attr-dict `:` type($dst) `(` $dstTy `)`
}];
let hasVerifier = 1;
let extraClassDeclaration = [{
static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::ConvertF32x2ToF6x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::ConvertF32x2ToF6x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$a, $b, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$a, $b, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2176,27 +2198,36 @@ class NVVM_ConvertFPx2ToF6x2Op<string srcType>
`dst` with 2 MSB bits padded with zeros.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector with 2 MSB bits padded with zeros.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
}];
let results = (outs AnyTypeOf<[I16, VectorOfLengthAndType<[2], [I8]>]>:$dst);
let arguments = (ins
VectorOfLengthAndType<[2], [!cast<Type>(srcType)]>:$src,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<AnyTypeOf<[F6E2M3FN, F6E3M2FN]>>:$dstTy);
+ TypeAttrOf<AnyTypeOf<[F6E2M3FN, F6E3M2FN]>>:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $src oilist(`relu` `=` $relu) attr-dict
+ $src oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
let extraClassDeclaration = [{
- static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy, bool hasRelu);
+ static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
+ bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::Convert}]#srcType#[{x2ToF6x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::Convert}]#srcType#[{x2ToF6x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$src, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$src, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2220,8 +2251,11 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
The `rnd` and `sat` attributes specify the rounding and saturation modes respectively.
+ Conversions to `f8E4M3FN` and `f8E5M2` support `rn` and `rz` rounding.
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction. `pzo` is not supported for
+ conversions to `f8E8M0FNU`.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2234,10 +2268,11 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
DefaultValuedAttr<FPRoundingModeAttr, "FPRoundingMode::NONE">:$rnd,
DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
$a `,` $b oilist(
- `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu
+ `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu | `pzo` `=` $pzo
) attr-dict `:` type($dst) `(` $dstTy `)`
}];
@@ -2252,7 +2287,7 @@ def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> {
auto intId = NVVM::ConvertF32x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $sat, $relu);
llvm::SmallVector<llvm::Value *> args = {$a, $b};
if (!mlir::isa<mlir::Float8E8M0FNUType>($dstTy))
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1($pzo));
llvm::Value *packedI16 = createIntrinsicCall(builder, intId, args);
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
@@ -2275,8 +2310,10 @@ def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> {
second element of `a` is stored in the lower 8 bits of `dst`.
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
+ The `rnd` attribute specifies the rounding mode (`rn` or `rz`).
The `relu` attribute, when set, lowers to the '.relu' variant of
- the cvt instruction.
+ the cvt instruction. The `pzo` attribute, when set, lowers to the
+ '.pzo' variant of the cvt instruction.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2286,21 +2323,27 @@ def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> {
let arguments = (ins
VectorOfLengthAndType<[2], [F16]>:$a,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttr:$dstTy);
+ TypeAttr:$dstTy,
+ DefaultValuedAttr<ConfinedAttr<FPRoundingModeAttr,
+ [EnumAttrIsOneOf<FPRoundingModeAttr,
+ [FPRoundingModeRN, FPRoundingModeRZ]>]>,
+ "FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
- $a oilist(`relu` `=` $relu) attr-dict
+ $a oilist(`rnd` `=` $rnd | `relu` `=` $relu | `pzo` `=` $pzo) attr-dict
`:` type($a) `->` type($dst) `(` $dstTy `)`
}];
let extraClassDeclaration = [{
static llvm::Intrinsic::ID getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd,
bool hasRelu);
}];
string llvmBuilder = [{
- auto intId = NVVM::ConvertF16x2ToF8x2Op::getIntrinsicID($dstTy, $relu);
+ auto intId = NVVM::ConvertF16x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $relu);
llvm::Value *packedI16 =
- createIntrinsicCall(builder, intId, {$a, builder.getInt1(false)});
+ createIntrinsicCall(builder, intId, {$a, builder.getInt1($pzo)});
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
else
@@ -2322,7 +2365,11 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
If `dst` is returned as a vector type, each converted value is stored as an
i8 element in the vector.
The `rnd` and `sat` attributes specify the rounding and saturation modes
- respectively.
+ respectively. Conversions to `f8E4M3FN` and `f8E5M2` support `rn` and `rz`
+ rounding. The `relu` attribute, when set, lowers to the '.relu' variant of
+ the cvt instruction. The `pzo` attribute, when set, lowers to the '.pzo'
+ variant of the cvt instruction. `pzo` is not supported for conversions to
+ `f8E8M0FNU`.
[For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)
}];
@@ -2334,10 +2381,11 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
DefaultValuedAttr<FPRoundingModeAttr, "FPRoundingMode::NONE">:$rnd,
DefaultValuedAttr<SaturationModeAttr, "SaturationMode::NONE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
- TypeAttrOf<AnyTypeOf<[F8E8M0FNU, F8E4M3FN, F8E5M2]>>:$dstTy);
+ TypeAttrOf<AnyTypeOf<[F8E8M0FNU, F8E4M3FN, F8E5M2]>>:$dstTy,
+ DefaultValuedAttr<BoolAttr, "false">:$pzo);
let assemblyFormat = [{
$src oilist(
- `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu
+ `rnd` `=` $rnd | `sat` `=` $sat | `relu` `=` $relu | `pzo` `=` $pzo
) attr-dict
`:` type($src) `->` type($dst) `(` $dstTy `)`
}];
@@ -2352,7 +2400,7 @@ def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> {
auto intId = NVVM::ConvertBF16x2ToF8x2Op::getIntrinsicID($dstTy, $rnd, $sat, $relu);
llvm::SmallVector<llvm::Value *> args = {$src};
if (!mlir::isa<mlir::Float8E8M0FNUType>($dstTy))
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1($pzo));
llvm::Value *packedI16 = createIntrinsicCall(builder, intId, args);
if(op.getDst().getType().isInteger(16))
$dst = packedI16;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 21c7c1cc148798..1a300b1e7f41c7 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -548,15 +548,16 @@ LogicalResult ConvertF32x2ToF8x2Op::verify() {
bool isSatFinite = getSat() == SatMode::SATFINITE;
bool hasRelu = getRelu();
+ bool hasPzo = getPzo();
mlir::MLIRContext *ctx = getContext();
return llvm::TypeSwitch<mlir::Type, LogicalResult>(getDstTy())
.Case<mlir::Float8E4M3FNType, mlir::Float8E5M2Type>(
[&](mlir::Type) -> LogicalResult {
- if (!isRoundingModeRN) {
- return emitOpError("Only RN rounding mode is supported for "
- "conversions from f32x2 to ")
+ if (!(isRoundingModeRN || isRoundingModeRZ)) {
+ return emitOpError("Only RN and RZ rounding modes are supported "
+ "for conversions from f32x2 to ")
<< mlir::Float8E4M3FNType::get(ctx) << " and "
<< mlir::Float8E5M2Type::get(ctx) << " types";
}
@@ -575,9 +576,10 @@ LogicalResult ConvertF32x2ToF8x2Op::verify() {
"conversions from f32x2 to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
}
- if (hasRelu) {
- return emitOpError("relu not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
+ if (hasRelu || hasPzo) {
+ return emitOpError() << (hasRelu ? "relu" : "pzo")
+ << " not supported for conversions to "
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
}
return success();
})
@@ -612,15 +614,16 @@ LogicalResult ConvertBF16x2ToF8x2Op::verify() {
bool isRoundingModeRP = getRnd() == RndMode::RP;
bool isSatFinite = getSat() == SatMode::SATFINITE;
bool hasRelu = getRelu();
+ bool hasPzo = getPzo();
mlir::MLIRContext *ctx = getContext();
return llvm::TypeSwitch<mlir::Type, LogicalResult>(getDstTy())
.Case<mlir::Float8E4M3FNType, mlir::Float8E5M2Type>(
[&](mlir::Type) -> LogicalResult {
- if (!isRoundingModeRN)
- return emitOpError("Only RN rounding mode is supported for "
- "conversions from bf16x2 to ")
+ if (!(isRoundingModeRN || isRoundingModeRZ))
+ return emitOpError("Only RN and RZ rounding modes are supported "
+ "for conversions from bf16x2 to ")
<< mlir::Float8E4M3FNType::get(ctx) << " and "
<< mlir::Float8E5M2Type::get(ctx) << " types";
if (!isSatFinite)
@@ -635,9 +638,10 @@ LogicalResult ConvertBF16x2ToF8x2Op::verify() {
return emitOpError("Only RZ and RP rounding modes are supported for "
"conversions from bf16x2 to ")
<< mlir::Float8E8M0FNUType::get(ctx) << " type";
- if (hasRelu)
- return emitOpError("relu not supported for conversions to ")
- << mlir::Float8E8M0FNUType::get(ctx) << " type";
+ if (hasRelu || hasPzo)
+ return emitOpError() << (hasRelu ? "relu" : "pzo")
+ << " not supported for conversions to "
+ << mlir::Float8E8M0FNUType::get(ctx) << " type";
return success();
})
.Default([&](mlir::Type) -> LogicalResult {
@@ -5086,30 +5090,42 @@ ConvertF32x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertF32x2ToF4x2Op op,
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getA()));
args.push_back(mt.lookupValue(op.getB()));
- args.push_back(builder.getInt1(false));
-
- bool hasRelu = op.getRelu();
+ args.push_back(builder.getInt1(op.getPzo()));
- llvm::Intrinsic::ID intId =
- hasRelu ? llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_satfinite;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m1x2_rz_relu_satfinite,
+ };
- return {intId, std::move(args)};
+ return {ids[idx], std::move(args)};
}
-#define GET_F32x2_TO_F6x2_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_ff_to_##type##_rn_relu_satfinite \
- : llvm::Intrinsic::nvvm_ff_to_##type##_rn_satfinite
+llvm::Intrinsic::ID
+ConvertF32x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_e3m2x2_rz_relu_satfinite,
+ };
-llvm::Intrinsic::ID ConvertF32x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float6E2M3FNType) {
- return GET_F32x2_TO_F6x2_ID(e2m3x2, hasRelu);
- })
- .Case([&](mlir::Float6E3M2FNType) {
- return GET_F32x2_TO_F6x2_ID(e3m2x2, hasRelu);
- })
+ .Case([&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case([&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF32x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
@@ -5121,17 +5137,24 @@ ConvertF16x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertF16x2ToF4x2Op &op,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
mlir::Type dstTy = op.getDstTy();
- bool hasRelu = op.getRelu();
- llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rz_relu_satfinite,
+ };
+ llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
if (llvm::isa<mlir::Float4E2M1FNType>(dstTy))
- intId = hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e2m1x2_rn_satfinite;
+ intId = ids[idx];
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getSrc()));
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1(op.getPzo()));
return {intId, std::move(args)};
}
@@ -5141,86 +5164,121 @@ ConvertBF16x2ToF4x2Op::getIntrinsicIDAndArgs(NVVM::ConvertBF16x2ToF4x2Op &op,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
mlir::Type dstTy = op.getDstTy();
- bool hasRelu = op.getRelu();
- llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
+ unsigned idx =
+ (static_cast<unsigned>(op.getRnd() == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(op.getRelu());
+ static constexpr llvm::Intrinsic::ID ids[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rz_relu_satfinite,
+ };
+ llvm::Intrinsic::ID intId = llvm::Intrinsic::not_intrinsic;
if (llvm::isa<mlir::Float4E2M1FNType>(dstTy))
- intId = hasRelu ? llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e2m1x2_rn_satfinite;
+ intId = ids[idx];
llvm::SmallVector<llvm::Value *> args;
args.push_back(mt.lookupValue(op.getSrc()));
- args.push_back(builder.getInt1(false));
+ args.push_back(builder.getInt1(op.getPzo()));
return {intId, std::move(args)};
}
-llvm::Intrinsic::ID ConvertF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
+llvm::Intrinsic::ID
+ConvertF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rz_relu_satfinite,
+ };
+
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float6E2M3FNType>([&](mlir::Float6E2M3FNType) {
- return hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e2m3x2_rn_satfinite;
- })
- .Case<mlir::Float6E3M2FNType>([&](mlir::Float6E3M2FNType) {
- return hasRelu ? llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_f16x2_to_e3m2x2_rn_satfinite;
- })
+ .Case<mlir::Float6E2M3FNType>(
+ [&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case<mlir::Float6E3M2FNType>(
+ [&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF16x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
});
}
-llvm::Intrinsic::ID ConvertBF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
+llvm::Intrinsic::ID
+ConvertBF16x2ToF6x2Op::getIntrinsicID(mlir::Type dstTy,
+ NVVM::FPRoundingMode rnd, bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e2m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e3m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rz_relu_satfinite,
+ };
+
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float6E2M3FNType>([&](mlir::Float6E2M3FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e2m3x2_rn_satfinite;
- })
- .Case<mlir::Float6E3M2FNType>([&](mlir::Float6E3M2FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e3m2x2_rn_satfinite;
- })
+ .Case<mlir::Float6E2M3FNType>(
+ [&](mlir::Float6E2M3FNType) { return e2m3x2IDs[idx]; })
+ .Case<mlir::Float6E3M2FNType>(
+ [&](mlir::Float6E3M2FNType) { return e3m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertBF16x2ToF6x2Op");
return llvm::Intrinsic::not_intrinsic;
});
}
-#define GET_F32x2_TO_F8X2_US_ID(rnd, has_satf) \
- has_satf ? llvm::Intrinsic::nvvm_ff_to_ue8m0x2_##rnd##_satfinite \
- : llvm::Intrinsic::nvvm_ff_to_ue8m0x2_##rnd
-
-#define GET_F32x2_TO_F8X2_S_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_ff_to_##type##_rn_relu \
- : llvm::Intrinsic::nvvm_ff_to_##type##_rn
-
llvm::Intrinsic::ID
ConvertF32x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
NVVM::SaturationMode sat, bool hasRelu) {
bool hasSatFinite = (sat == NVVM::SaturationMode::SATFINITE);
- bool hasRoundingModeRZ = (rnd == NVVM::FPRoundingMode::RZ);
- bool hasRoundingModeRP = (rnd == NVVM::FPRoundingMode::RP);
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ unsigned ue8m0Idx = (static_cast<unsigned>(hasSatFinite) << 1) |
+ static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RP);
+
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rn,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rn_relu,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_e4m3x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rn,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rn_relu,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_e5m2x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID ue8m0x2IDs[] = {
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rz,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rp,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_ff_to_ue8m0x2_rp_satfinite,
+ };
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float8E4M3FNType) {
- return GET_F32x2_TO_F8X2_S_ID(e4m3x2, hasRelu);
- })
- .Case([&](mlir::Float8E5M2Type) {
- return GET_F32x2_TO_F8X2_S_ID(e5m2x2, hasRelu);
- })
+ .Case([&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case([&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
.Case([&](mlir::Float8E8M0FNUType) {
- if (hasRoundingModeRZ)
- return GET_F32x2_TO_F8X2_US_ID(rz, hasSatFinite);
- else if (hasRoundingModeRP)
- return GET_F32x2_TO_F8X2_US_ID(rp, hasSatFinite);
-
- llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
+ if (rnd != NVVM::FPRoundingMode::RZ && rnd != NVVM::FPRoundingMode::RP)
+ llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
+ return ue8m0x2IDs[ue8m0Idx];
})
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF32x2ToF8x2Op");
@@ -5228,19 +5286,27 @@ ConvertF32x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
});
}
-#define GET_F16x2_TO_F8X2_ID(type, has_relu) \
- has_relu ? llvm::Intrinsic::nvvm_f16x2_to_##type##_rn_relu \
- : llvm::Intrinsic::nvvm_f16x2_to_##type##_rn
+llvm::Intrinsic::ID
+ConvertF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy, NVVM::FPRoundingMode rnd,
+ bool hasRelu) {
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rn,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rn_relu,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rz,
+ llvm::Intrinsic::nvvm_f16x2_to_e4m3x2_rz_relu,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rn,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rn_relu,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rz,
+ llvm::Intrinsic::nvvm_f16x2_to_e5m2x2_rz_relu,
+ };
-llvm::Intrinsic::ID ConvertF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
- bool hasRelu) {
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case([&](mlir::Float8E4M3FNType) {
- return GET_F16x2_TO_F8X2_ID(e4m3x2, hasRelu);
- })
- .Case([&](mlir::Float8E5M2Type) {
- return GET_F16x2_TO_F8X2_ID(e5m2x2, hasRelu);
- })
+ .Case([&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case([&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertF16x2ToF8x2Op");
return llvm::Intrinsic::not_intrinsic;
@@ -5252,7 +5318,23 @@ ConvertBF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
NVVM::FPRoundingMode rnd,
NVVM::SaturationMode sat, bool hasRelu) {
bool hasSatFinite = (sat == NVVM::SaturationMode::SATFINITE);
-
+ unsigned idx = (static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RZ) << 1) |
+ static_cast<unsigned>(hasRelu);
+ unsigned ue8m0Idx = (static_cast<unsigned>(hasSatFinite) << 1) |
+ static_cast<unsigned>(rnd == NVVM::FPRoundingMode::RP);
+
+ static constexpr llvm::Intrinsic::ID e4m3x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rz_relu_satfinite,
+ };
+ static constexpr llvm::Intrinsic::ID e5m2x2IDs[] = {
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_relu_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rz_satfinite,
+ llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rz_relu_satfinite,
+ };
static constexpr llvm::Intrinsic::ID ue8m0x2IDs[] = {
llvm::Intrinsic::nvvm_bf16x2_to_ue8m0x2_rz,
llvm::Intrinsic::nvvm_bf16x2_to_ue8m0x2_rp,
@@ -5261,21 +5343,12 @@ ConvertBF16x2ToF8x2Op::getIntrinsicID(mlir::Type dstTy,
};
return llvm::TypeSwitch<mlir::Type, llvm::Intrinsic::ID>(dstTy)
- .Case<mlir::Float8E4M3FNType>([&](mlir::Float8E4M3FNType) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e4m3x2_rn_satfinite;
- })
- .Case<mlir::Float8E5M2Type>([&](mlir::Float8E5M2Type) {
- return hasRelu
- ? llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_relu_satfinite
- : llvm::Intrinsic::nvvm_bf16x2_to_e5m2x2_rn_satfinite;
- })
- .Case<mlir::Float8E8M0FNUType>([&](mlir::Float8E8M0FNUType) {
- bool hasRoundingModeRP = (rnd == NVVM::FPRoundingMode::RP);
- unsigned index = (hasSatFinite << 1) | hasRoundingModeRP;
- return ue8m0x2IDs[index];
- })
+ .Case<mlir::Float8E4M3FNType>(
+ [&](mlir::Float8E4M3FNType) { return e4m3x2IDs[idx]; })
+ .Case<mlir::Float8E5M2Type>(
+ [&](mlir::Float8E5M2Type) { return e5m2x2IDs[idx]; })
+ .Case<mlir::Float8E8M0FNUType>(
+ [&](mlir::Float8E8M0FNUType) { return ue8m0x2IDs[ue8m0Idx]; })
.Default([](mlir::Type) {
llvm_unreachable("Invalid conversion in ConvertBF16x2ToF8x2Op");
return llvm::Intrinsic::not_intrinsic;
diff --git a/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir b/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
index 608622224e8cc3..d9d293a96d6052 100644
--- a/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
+++ b/mlir/test/Dialect/LLVMIR/nvvm/invalid-convert-stochastic-rounding.mlir
@@ -56,7 +56,7 @@ llvm.func @convert_float_to_tf32_rs_not_supported(%src : f32) -> i32 {
// -----
llvm.func @convert_f32x2_to_f8x2_rs_not_supported(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rs> sat = <satfinite> : i16 (f8E4M3FN)
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
index 9538b7d798f7ae..4a0401960298a1 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2.mlir
@@ -37,6 +37,75 @@ llvm.func @convert_bf16x2_to_f4x2(%srcA : vector<2xbf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_f4x2_rz_pzo
+llvm.func @convert_f32x2_to_f4x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> : i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> relu = true : i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rn.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rn.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB relu = true pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> pzo = true : i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.ff.to.e2m1x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.f32x2.to.f4x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i8 (f4E2M1FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f16x2_to_f4x2_rz_pzo
+llvm.func @convert_f16x2_to_f4x2_rz_pzo(%srcA : vector<2xf16>) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> relu = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rn.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.f16x2.to.f4x2 %srcA pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rn.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.f16x2.to.f4x2 %srcA relu = true pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.f16x2.to.e2m1x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.f16x2.to.f4x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_bf16x2_to_f4x2_rz_pzo
+llvm.func @convert_bf16x2_to_f4x2_rz_pzo(%srcA : vector<2xbf16>) {
+ // CHECK: %[[res1:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res1]] to i8
+ %res1 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res2:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res2]] to i8
+ %res2 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> relu = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res3:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res3]] to i8
+ %res3 = nvvm.convert.bf16x2.to.f4x2 %srcA pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res4:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res4]] to i8
+ %res4 = nvvm.convert.bf16x2.to.f4x2 %srcA relu = true pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res5:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res5]] to i8
+ %res5 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ // CHECK: %[[res6:.*]] = call i16 @llvm.nvvm.bf16x2.to.e2m1x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ // CHECK-NEXT: %{{.*}} = trunc i16 %[[res6]] to i8
+ %res6 = nvvm.convert.bf16x2.to.f4x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f4x2_to_f16x2
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
index d179431fe93699..88b7a46d387f95 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp4x2_invalid.mlir
@@ -15,3 +15,19 @@ llvm.func @convert_bf16x2_to_f4x2_invalid_type(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f4x2 %src : vector<2xbf16> -> i8 (f8E4M3FN)
llvm.return
}
+
+// -----
+
+llvm.func @convert_f32x2_to_f4x2_invalid_rounding(%a : f32, %b : f32) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f32x2.to.f4x2 %a, %b rnd = <rp> : i8 (f4E2M1FN)
+ llvm.return
+}
+
+// -----
+
+llvm.func @convert_bf16x2_to_f4x2_invalid_rounding(%src : vector<2xbf16>) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.bf16x2.to.f4x2 %src rnd = <rs> : vector<2xbf16> -> i8 (f4E2M1FN)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
index 21f23521405b1d..cd6cd27471ded7 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2.mlir
@@ -91,6 +91,72 @@ llvm.func @convert_bf16x2_to_fp6x2_vector(%srcA : vector<2xbf16>, %scale_factor
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_fp6x2_rz_pzo
+llvm.func @convert_f32x2_to_fp6x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rn.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB relu = true pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> pzo = true : i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e2m3x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e3m2x2.rz.satfinite(float %{{.*}}, float %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> : i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e3m2x2.rz.relu.satfinite(float %{{.*}}, float %{{.*}}, i1 true)
+ %res8 = nvvm.convert.f32x2.to.f6x2 %srcA, %srcB rnd = <rz> relu = true pzo = true : i16 (f6E3M2FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_f16x2_to_fp6x2_rz_pzo
+llvm.func @convert_f16x2_to_fp6x2_rz_pzo(%srcA : vector<2xf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rn.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f16x2.to.f6x2 %srcA pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rn.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f16x2.to.f6x2 %srcA relu = true pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e2m3x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e3m2x2.rz.satfinite(<2 x half> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> : vector<2xf16> -> i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e3m2x2.rz.relu.satfinite(<2 x half> %{{.*}}, i1 true)
+ %res8 = nvvm.convert.f16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
+
+// CHECK-LABEL: @convert_bf16x2_to_fp6x2_rz_pzo
+llvm.func @convert_bf16x2_to_fp6x2_rz_pzo(%srcA : vector<2xbf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.bf16x2.to.f6x2 %srcA pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.bf16x2.to.f6x2 %srcA relu = true pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e2m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i16 (f6E2M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e3m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> : vector<2xbf16> -> i16 (f6E3M2FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e3m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res8 = nvvm.convert.bf16x2.to.f6x2 %srcA rnd = <rz> relu = true pzo = true : vector<2xbf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f6x2_to_f16x2_e2m3
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
index e993868cf1c9f8..52904807a484be 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp6x2_invalid.mlir
@@ -15,3 +15,19 @@ llvm.func @convert_bf16x2_to_f6x2_invalid_type(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f6x2 %src : vector<2xbf16> -> vector<2xi8> (f8E4M3FN)
llvm.return
}
+
+// -----
+
+llvm.func @convert_f32x2_to_f6x2_invalid_rounding(%a : f32, %b : f32) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f32x2.to.f6x2 %a, %b rnd = <rp> : i16 (f6E2M3FN)
+ llvm.return
+}
+
+// -----
+
+llvm.func @convert_f16x2_to_f6x2_invalid_rounding(%src : vector<2xf16>) {
+ // expected-error @below {{attribute 'rnd' failed to satisfy constraint: NVVM FPRoundingMode kind whose value is one of {rn, rz}}}
+ %res = nvvm.convert.f16x2.to.f6x2 %src rnd = <rm> : vector<2xf16> -> i16 (f6E3M2FN)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
index 058e28be90283c..750b81c4843e3f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2.mlir
@@ -44,6 +44,36 @@ llvm.func @convert_f32x2_to_f8x2_vector_return(%srcA : f32, %srcB : f32) {
llvm.return
}
+// CHECK-LABEL: @convert_f32x2_to_f8x2_rz_pzo
+llvm.func @convert_f32x2_to_f8x2_rz_pzo(%srcA : f32, %srcB : f32) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz(float %{{.*}}, float %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float %{{.*}}, float %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> relu = true pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz(float %{{.*}}, float %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> pzo = true : i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e4m3x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true pzo = true : i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz(float %{{.*}}, float %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 false)
+ %res8 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float %{{.*}}, float %{{.*}}, i1 true)
+ %res9 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res10 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rn> sat = <satfinite> relu = true pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz(float %{{.*}}, float %{{.*}}, i1 true)
+ %res11 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> pzo = true : i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.ff.to.e5m2x2.rz.relu(float %{{.*}}, float %{{.*}}, i1 true)
+ %res12 = nvvm.convert.f32x2.to.f8x2 %srcA, %srcB rnd = <rz> sat = <satfinite> relu = true pzo = true : i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f16x2_to_f8x2_e4m3
@@ -75,6 +105,36 @@ llvm.func @convert_f16x2_to_f8x2_vector_return(%src : vector<2xf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_f16x2_to_f8x2_rz_pzo
+llvm.func @convert_f16x2_to_f8x2_rz_pzo(%src : vector<2xf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz(<2 x half> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz.relu(<2 x half> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.f16x2.to.f8x2 %src pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.f16x2.to.f8x2 %src relu = true pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz(<2 x half> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rz.relu(<2 x half> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz(<2 x half> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz.relu(<2 x half> %{{.*}}, i1 false)
+ %res8 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> %{{.*}}, i1 true)
+ %res9 = nvvm.convert.f16x2.to.f8x2 %src pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> %{{.*}}, i1 true)
+ %res10 = nvvm.convert.f16x2.to.f8x2 %src relu = true pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz(<2 x half> %{{.*}}, i1 true)
+ %res11 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rz.relu(<2 x half> %{{.*}}, i1 true)
+ %res12 = nvvm.convert.f16x2.to.f8x2 %src rnd = <rz> relu = true pzo = true : vector<2xf16> -> i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_bf16x2_to_f8x2_ue8m0
@@ -126,6 +186,36 @@ llvm.func @convert_bf16x2_to_f8x2_vector_return(%src : vector<2xbf16>) {
llvm.return
}
+// CHECK-LABEL: @convert_bf16x2_to_f8x2_rz_pzo
+llvm.func @convert_bf16x2_to_f8x2_rz_pzo(%src : vector<2xbf16>) {
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res1 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res2 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res3 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res4 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res5 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e4m3x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res6 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E4M3FN)
+
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res7 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 false)
+ %res8 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rn.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res9 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rn.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res10 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rn> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res11 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ // CHECK: %{{.*}} = call i16 @llvm.nvvm.bf16x2.to.e5m2x2.rz.relu.satfinite(<2 x bfloat> %{{.*}}, i1 true)
+ %res12 = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> sat = <satfinite> relu = true pzo = true : vector<2xbf16> -> i16 (f8E5M2)
+ llvm.return
+}
+
// -----
// CHECK-LABEL: @convert_f8x2_to_f16x2
diff --git a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
index 02a7600ec8304b..b12a0a586dc074 100644
--- a/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/convert_fp8x2_invalid.mlir
@@ -11,7 +11,7 @@ llvm.func @convert_bf16x2_to_f8x2_invalid_type(%src : vector<2xbf16>) {
// -----
llvm.func @convert_bf16x2_to_f8x2_invalid_rounding_1(%src : vector<2xbf16>) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from bf16x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from bf16x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rm> : vector<2xbf16> -> vector<2xi8> (f8E4M3FN)
llvm.return
}
@@ -39,3 +39,11 @@ llvm.func @convert_bf16x2_to_f8x2_invalid_relu(%src : vector<2xbf16>) {
%res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rp> relu = true : vector<2xbf16> -> vector<2xi8> (f8E8M0FNU)
llvm.return
}
+
+// -----
+
+llvm.func @convert_bf16x2_to_f8x2_invalid_pzo(%src : vector<2xbf16>) {
+ // expected-error @below {{pzo not supported for conversions to 'f8E8M0FNU' type}}
+ %res = nvvm.convert.bf16x2.to.f8x2 %src rnd = <rz> pzo = true : vector<2xbf16> -> vector<2xi8> (f8E8M0FNU)
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 9027f760d3483e..30e3ada49c6b13 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -141,15 +141,15 @@ llvm.func @nvvm_match_sync_any(%val32: i32, %thread_mask: i32) {
// -----
llvm.func @nvvm_cvt_float_to_f8x2_invalid_rounding_e4m3(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
- %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rz> sat = <satfinite> : i16 (f8E4M3FN)
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rm> sat = <satfinite> : i16 (f8E4M3FN)
llvm.return
}
// -----
llvm.func @nvvm_cvt_float_to_f8x2_invalid_rounding_e5m2(%a : f32, %b : f32) {
- // expected-error @below {{Only RN rounding mode is supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
+ // expected-error @below {{Only RN and RZ rounding modes are supported for conversions from f32x2 to 'f8E4M3FN' and 'f8E5M2' types}}
%res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rp> sat = <satfinite> : i16 (f8E5M2)
llvm.return
}
@@ -188,6 +188,14 @@ llvm.func @nvvm_cvt_float_to_f8x2_relu_not_supported_ue8m0(%a : f32, %b : f32) {
// -----
+llvm.func @nvvm_cvt_float_to_f8x2_pzo_not_supported_ue8m0(%a : f32, %b : f32) {
+ // expected-error @below {{pzo not supported for conversions to 'f8E8M0FNU' type}}
+ %res = nvvm.convert.f32x2.to.f8x2 %a, %b rnd = <rz> pzo = true : i16 (f8E8M0FNU)
+ llvm.return
+}
+
+// -----
+
llvm.func @nvvm_cvt_f16x2_to_f8x2_invalid_type(%src : vector<2xf16>) {
// expected-error @below {{Only 'f8E4M3FN' and 'f8E5M2' types are supported for conversions from f16x2 to f8x2.}}
%res = nvvm.convert.f16x2.to.f8x2 %src : vector<2xf16> -> i16 (f8E8M0FNU)
More information about the Mlir-commits
mailing list