[Mlir-commits] [mlir] [MLIR][NVGPU] Add truncf and extf Ops (PR #199700)
Srinivasa Ravi
llvmlistbot at llvm.org
Thu Jul 9 04:27:58 PDT 2026
https://github.com/Wolfram70 updated https://github.com/llvm/llvm-project/pull/199700
>From 1246175402141857ea45fec146e1a9f35770d2e5 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 26 May 2026 10:09:58 +0000
Subject: [PATCH 01/11] [MLIR][NVGPU] Add convert.fpext and convert.fptrunc Ops
This change adds the `convert.fpext` and `convert.fptrunc` Ops to the
NVGPU dialect to support floating-point conversion operations.
These Ops support scalar, vector and tensor inputs and lower to the
corresponding NVVM Ops after padding and chunking the input into i32
registers.
For tensor inputs, the tensors must be converted to vectors by other means
before they can be successfully lowered.
---
mlir/include/mlir/Conversion/Passes.td | 3 +-
mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 14 +
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 76 ++
.../lib/Conversion/NVGPUToNVVM/CMakeLists.txt | 1 +
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 670 ++++++++++++++++++
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 156 ++++
.../NVGPUToNVVM/nvgpu-convert-fpext.mlir | 466 ++++++++++++
.../NVGPUToNVVM/nvgpu-convert-fptrunc.mlir | 327 +++++++++
.../NVGPU/nvgpu-convert-fpext-invalid.mlir | 106 +++
.../NVGPU/nvgpu-convert-fptrunc-invalid.mlir | 126 ++++
10 files changed, 1944 insertions(+), 1 deletion(-)
create mode 100644 mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
create mode 100644 mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
create mode 100644 mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
create mode 100644 mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index b92bc1cd31723..a675ce431f567 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1106,7 +1106,8 @@ def ConvertNVGPUToNVVMPass : Pass<"convert-nvgpu-to-nvvm"> {
"arith::ArithDialect",
"LLVM::LLVMDialect",
"memref::MemRefDialect",
- "NVVM::NVVMDialect"
+ "NVVM::NVVMDialect",
+ "vector::VectorDialect"
];
}
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index 1c0d7bd1113ea..e56aca779cd1e 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -101,6 +101,20 @@ def TensorMapInterleaveKind : I32EnumAttr<"TensorMapInterleaveKind",
let cppNamespace = "::mlir::nvgpu";
}
+def SubBytesPackedCompact : I32EnumAttrCase<"COMPACT", 0, "compact">;
+def SubBytesPackedU6UnpackU8E3M2 : I32EnumAttrCase<"U6_UNPACK_U8_E3M2", 1, "u6_unpack_u8_e3m2">;
+def SubBytesPackedU6UnpackU8E2M3 : I32EnumAttrCase<"U6_UNPACK_U8_E2M3", 2, "u6_unpack_u8_e2m3">;
+def SubBytesPackedKind : I32EnumAttr<"SubBytesPackedKind",
+ "Sub-bytes packed kind type",
+ [SubBytesPackedCompact, SubBytesPackedU6UnpackU8E3M2,
+ SubBytesPackedU6UnpackU8E2M3]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::nvgpu";
+}
+def SubBytesPackedKindAttr : EnumAttr<NVGPU_Dialect, SubBytesPackedKind, "subbytes_packedkind"> {
+ let assemblyFormat = "`<` $value `>`";
+}
+
def RcpApprox : I32EnumAttrCase<"APPROX", 0, "approx">;
def RcpRN : I32EnumAttrCase<"RN", 1, "rn">;
def RcpRZ : I32EnumAttrCase<"RZ", 2, "rz">;
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 5a1771eecd0f6..07408ac18f95b 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -671,4 +671,80 @@ def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
let hasVerifier = 1;
}
+//===----------------------------------------------------------------------===//
+// NVGPU Conversion Ops
+//===----------------------------------------------------------------------===//
+
+def Int8OrFloatLike : TypeConstraint<
+ Or<[FloatLike.predicate,
+ I8.predicate,
+ ValueSemanticsContainerOf<[I8]>.predicate]>,
+ "scalar, vector, or tensor of i8 or floats">;
+def AnyI32Like : TypeOrValueSemanticsContainer<I32, "scalar i32 or vector of i32">;
+
+def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
+ [Pure]> {
+ let summary = "Truncate floating-point to narrower floating-point";
+ let description = [{
+ Truncate a floating-point value to a smaller floating-point type.
+ Destination must be strictly narrower than source.
+
+ Supported paths: f32->f16, f32->bf16, f32->f8, f32->f6, f32->f4,
+ f16->f8, f16->f4, bf16->f8, bf16->f4.
+
+ The `random_bits` operand enables stochastic rounding (RS mode)
+ for f32->f16/bf16 conversions. When provided, `rnd` must be RS.
+
+ For f6 types (f6E3M2FN/f6E2M3FN), result is i8 with 2-bit MSB
+ padding; use `packed_kind` to specify the f6 variant. `compact` is not
+ a supported `packed_kind` for f6 types.
+
+ Example:
+ ```mlir
+ %r = nvgpu.cvt_fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %r = nvgpu.cvt_fptrunc %in : f32 to f16
+ %r = nvgpu.cvt_fptrunc %in : vector<2x4xf16> to vector<2x4xf8E5M2>
+ ```
+ }];
+ let arguments = (ins FloatLike:$in,
+ DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<SubBytesPackedKindAttr, "SubBytesPackedKind::COMPACT">:$packed_kind,
+ DefaultValuedAttr<SaturationModeAttr, "NVVM::SaturationMode::SATFINITE">:$sat,
+ DefaultValuedAttr<BoolAttr, "false">:$relu,
+ Optional<I32>:$random_bits
+ );
+ let results = (outs Int8OrFloatLike:$out);
+ let assemblyFormat = "$in (`,` $random_bits^)? attr-dict `:` type($in) `to` type($out)";
+ let hasVerifier = 1;
+}
+
+def NVGPU_FPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
+ let summary = "Extend floating-point to wider floating-point";
+ let description = [{
+ Extend a floating-point value to a wider floating-point type.
+ Destination must be strictly wider than source.
+
+ Supported paths: f8->f16, f8->bf16, f6->f16, f4->f16, f16->f32,
+ bf16->f32. Narrow->f32 uses two-step lowering (NVVM op to f16/bf16
+ intermediate, then LLVM FPExt). e8m0->bf16 and e8m0->f32 supported
+ (f32 goes through bf16 intermediate).
+
+ For f6 types, input is i8 with `packed_kind` specifying the f6 variant.
+
+ Example:
+ ```mlir
+ %r = nvgpu.cvt_fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %r = nvgpu.cvt_fpext %in : vector<4xf8E5M2> to vector<4xf32>
+ %r = nvgpu.cvt_fpext %in : f8E4M3FN to f32
+ ```
+ }];
+ let arguments = (ins Int8OrFloatLike:$in,
+ DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<SubBytesPackedKindAttr, "SubBytesPackedKind::COMPACT">:$packed_kind,
+ DefaultValuedAttr<BoolAttr, "false">:$relu);
+ let results = (outs FloatLike:$out);
+ let assemblyFormat = "$in attr-dict `:` type($in) `to` type($out)";
+ let hasVerifier = 1;
+}
+
#endif // MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/CMakeLists.txt b/mlir/lib/Conversion/NVGPUToNVVM/CMakeLists.txt
index a050749eb7da8..2e0a417bd823d 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/NVGPUToNVVM/CMakeLists.txt
@@ -18,6 +18,7 @@ add_mlir_conversion_library(MLIRNVGPUToNVVM
MLIRNVGPUDialect
MLIRNVVMDialect
MLIRArithDialect
+ MLIRVectorDialect
MLIRPass
MLIRSCFTransforms
MLIRTransforms
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index e566449ffadff..8b992b67aed29 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -20,6 +20,7 @@
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
#include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/TypeUtilities.h"
@@ -461,6 +462,7 @@ struct ConvertNVGPUToNVVMPass
target.addLegalDialect<::mlir::arith::ArithDialect>();
target.addLegalDialect<::mlir::memref::MemRefDialect>();
target.addLegalDialect<::mlir::NVVM::NVVMDialect>();
+ target.addLegalDialect<::mlir::vector::VectorDialect>();
mlir::scf::populateSCFStructuralTypeConversionsAndLegality(
converter, patterns, target);
if (failed(applyPartialConversion(getOperation(), target,
@@ -1709,6 +1711,669 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
rewriter);
}
};
+
+//===----------------------------------------------------------------------===//
+// FPTruncOp Lowering
+//===----------------------------------------------------------------------===//
+
+/// Conversion op identifier for nvgpu.convert.fptrunc lowering dispatch table.
+enum class TruncConvOp {
+ F32x2_TO_F16x2,
+ F32x2_TO_BF16x2,
+ F32x2_TO_F8x2,
+ F32x2_TO_F6x2,
+ F32x2_TO_F4x2,
+ F16x2_TO_F8x2,
+ F16x2_TO_F4x2,
+ BF16x2_TO_F8x2,
+ BF16x2_TO_F4x2,
+};
+
+enum class TruncSrcKind { F32, F16, BF16 };
+
+enum class TruncDstKind { F16, BF16, F8, F6, F4 };
+
+struct TruncTableEntry {
+ TruncSrcKind src;
+ TruncDstKind dst;
+ TruncConvOp convOp;
+ int srcStepDecrement; // 2 for f32 pairs, 1 for f16x2/bf16x2
+};
+
+static constexpr TruncTableEntry kTruncTable[] = {
+ // f32 source
+ {TruncSrcKind::F32, TruncDstKind::F16, TruncConvOp::F32x2_TO_F16x2, 2},
+ {TruncSrcKind::F32, TruncDstKind::BF16, TruncConvOp::F32x2_TO_BF16x2, 2},
+ {TruncSrcKind::F32, TruncDstKind::F8, TruncConvOp::F32x2_TO_F8x2, 2},
+ {TruncSrcKind::F32, TruncDstKind::F6, TruncConvOp::F32x2_TO_F6x2, 2},
+ {TruncSrcKind::F32, TruncDstKind::F4, TruncConvOp::F32x2_TO_F4x2, 2},
+ // f16 source
+ {TruncSrcKind::F16, TruncDstKind::F8, TruncConvOp::F16x2_TO_F8x2, 1},
+ {TruncSrcKind::F16, TruncDstKind::F4, TruncConvOp::F16x2_TO_F4x2, 1},
+ // bf16 source
+ {TruncSrcKind::BF16, TruncDstKind::F8, TruncConvOp::BF16x2_TO_F8x2, 1},
+ {TruncSrcKind::BF16, TruncDstKind::F4, TruncConvOp::BF16x2_TO_F4x2, 1},
+};
+
+static bool isConvertibleF8Type(Type t) {
+ return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(t);
+}
+
+static std::optional<TruncSrcKind> classifySrcType(Type t) {
+ if (t.isF32())
+ return TruncSrcKind::F32;
+ if (t.isF16())
+ return TruncSrcKind::F16;
+ if (t.isBF16())
+ return TruncSrcKind::BF16;
+ return std::nullopt;
+}
+
+static std::optional<TruncDstKind> classifyDstType(Type t) {
+ if (t.isF16())
+ return TruncDstKind::F16;
+ if (t.isBF16())
+ return TruncDstKind::BF16;
+ if (isConvertibleF8Type(t))
+ return TruncDstKind::F8;
+ int bitWidth = t.getIntOrFloatBitWidth();
+ if (isa<IntegerType>(t) && bitWidth == 8)
+ return TruncDstKind::F6;
+ if (bitWidth == 4)
+ return TruncDstKind::F4;
+ return std::nullopt;
+}
+
+static std::optional<std::pair<TruncConvOp, int>>
+lookupTruncConvOp(Type srcElemType, Type dstElemType) {
+ auto srcKind = classifySrcType(srcElemType);
+ auto dstKind = classifyDstType(dstElemType);
+ if (!srcKind || !dstKind)
+ return std::nullopt;
+ for (const auto &entry : kTruncTable) {
+ if (entry.src == *srcKind && entry.dst == *dstKind)
+ return {{entry.convOp, entry.srcStepDecrement}};
+ }
+ return std::nullopt;
+}
+
+static Value extractElement(ImplicitLocOpBuilder &b, Value srcVec, int idx) {
+ IntegerType i64Ty = b.getI64Type();
+ return b.create<LLVM::ExtractElementOp>(
+ srcVec, b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx)));
+}
+
+/// Extract a pair of f32 values from an i32 vector at the given base index.
+/// Returns {f32_lo (lower index), f32_hi (higher index)}.
+static std::pair<Value, Value> extractF32Pair(ImplicitLocOpBuilder &b,
+ Value srcI32Vec, int baseIdx) {
+ FloatType f32Ty = b.getF32Type();
+ Value elem0 = extractElement(b, srcI32Vec, baseIdx);
+ Value elem1 = extractElement(b, srcI32Vec, baseIdx + 1);
+ return {b.create<LLVM::BitcastOp>(f32Ty, elem0),
+ b.create<LLVM::BitcastOp>(f32Ty, elem1)};
+}
+
+static Value extractAndBitcast(ImplicitLocOpBuilder &b, Value srcI32Vec,
+ int idx, VectorType vecTy) {
+ Value elem = extractElement(b, srcI32Vec, idx);
+ return b.create<LLVM::BitcastOp>(vecTy, elem);
+}
+
+/// Dispatch to the specific NVVM conversion op based on TruncConvOp,
+/// extract source operands, and return the native result.
+/// - f16/bf16 destinations: returns i32 (bitcast from vector<2xf16/bf16>)
+/// - f8/f6 destinations: returns i16 (packed 2 x f8/f6)
+/// - f4 destinations: returns i8 (packed 2 x f4)
+/// Create a sub-byte conversion from an f32 pair source.
+template <typename ConvertOp, typename... Args>
+static Value convertFromF32Pair(ImplicitLocOpBuilder &b, Value srcI32Vec,
+ int srcBaseIdx, Type resultTy, Args &&...args) {
+ auto [lo, hi] = extractF32Pair(b, srcI32Vec, srcBaseIdx);
+ return b.create<ConvertOp>(resultTy, hi, lo, std::forward<Args>(args)...);
+}
+
+/// Create a sub-byte conversion from a packed f16x2/bf16x2 source.
+template <typename ConvertOp, typename... Args>
+static Value convertFromPacked(ImplicitLocOpBuilder &b, Value srcI32Vec,
+ int srcBaseIdx, Type srcElemTy, Type resultTy,
+ Args &&...args) {
+ Value src = extractAndBitcast(b, srcI32Vec, srcBaseIdx,
+ VectorType::get(2, srcElemTy));
+ return b.create<ConvertOp>(resultTy, src, std::forward<Args>(args)...);
+}
+
+static Value createTruncConversion(
+ ImplicitLocOpBuilder &b, MLIRContext *ctx, TruncConvOp convOp,
+ Value srcI32Vec, int srcBaseIdx, NVVM::FPRoundingModeAttr rndAttr,
+ NVVM::SaturationModeAttr satAttr, BoolAttr reluAttr, Type dstElemType,
+ Type actualDstFloatType, Value randomBits = Value()) {
+ IntegerType i8Ty = b.getI8Type();
+ IntegerType i16Ty = b.getI16Type();
+ IntegerType i32Ty = b.getI32Type();
+ auto dstTyAttr = TypeAttr::get(dstElemType);
+ auto actualDstTyAttr = TypeAttr::get(actualDstFloatType);
+
+ switch (convOp) {
+ case TruncConvOp::F32x2_TO_F16x2: {
+ auto [lo, hi] = extractF32Pair(b, srcI32Vec, srcBaseIdx);
+ Value r = b.create<NVVM::ConvertF32x2ToF16x2Op>(
+ VectorType::get(2, b.getF16Type()), hi, lo, randomBits, rndAttr,
+ satAttr, reluAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case TruncConvOp::F32x2_TO_BF16x2: {
+ auto [lo, hi] = extractF32Pair(b, srcI32Vec, srcBaseIdx);
+ Value r = b.create<NVVM::ConvertF32x2ToBF16x2Op>(
+ VectorType::get(2, b.getBF16Type()), hi, lo, randomBits, rndAttr,
+ satAttr, reluAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case TruncConvOp::F32x2_TO_F8x2:
+ return convertFromF32Pair<NVVM::ConvertF32x2ToF8x2Op>(
+ b, srcI32Vec, srcBaseIdx, i16Ty, rndAttr, satAttr, reluAttr, dstTyAttr);
+ case TruncConvOp::F32x2_TO_F6x2:
+ return convertFromF32Pair<NVVM::ConvertF32x2ToF6x2Op>(
+ b, srcI32Vec, srcBaseIdx, i16Ty, reluAttr, actualDstTyAttr);
+ case TruncConvOp::F32x2_TO_F4x2:
+ return convertFromF32Pair<NVVM::ConvertF32x2ToF4x2Op>(
+ b, srcI32Vec, srcBaseIdx, i8Ty, reluAttr, dstTyAttr);
+ case TruncConvOp::F16x2_TO_F8x2:
+ return convertFromPacked<NVVM::ConvertF16x2ToF8x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getF16Type(), i16Ty, reluAttr, dstTyAttr);
+ case TruncConvOp::F16x2_TO_F4x2:
+ return convertFromPacked<NVVM::ConvertF16x2ToF4x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getF16Type(), i8Ty, reluAttr,
+ actualDstTyAttr);
+ case TruncConvOp::BF16x2_TO_F8x2:
+ return convertFromPacked<NVVM::ConvertBF16x2ToF8x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getBF16Type(), i16Ty, rndAttr, satAttr,
+ reluAttr, dstTyAttr);
+ case TruncConvOp::BF16x2_TO_F4x2:
+ return convertFromPacked<NVVM::ConvertBF16x2ToF4x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getBF16Type(), i8Ty, reluAttr,
+ actualDstTyAttr);
+ }
+ llvm_unreachable("unhandled TruncConvOp");
+}
+
+struct NVGPUFPTruncOpLowering
+ : public ConvertOpToLLVMPattern<nvgpu::FPTruncOp> {
+ using ConvertOpToLLVMPattern<nvgpu::FPTruncOp>::ConvertOpToLLVMPattern;
+
+ static Type getActualDstFloatType(MLIRContext *ctx, Type elemType,
+ nvgpu::SubBytesPackedKind packedKind) {
+ if (isa<IntegerType>(elemType)) {
+ if (packedKind == nvgpu::SubBytesPackedKind::U6_UNPACK_U8_E3M2)
+ return Float6E3M2FNType::get(ctx);
+ return Float6E2M3FNType::get(ctx);
+ }
+ return elemType;
+ }
+
+ LogicalResult
+ matchAndRewrite(nvgpu::FPTruncOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ // Tensor inputs are not handled here; they must be converted to vectors
+ // by other means before this pattern runs.
+ if (isa<RankedTensorType>(op.getIn().getType()))
+ return rewriter.notifyMatchFailure(
+ op, "tensor inputs not handled; type converter should lower first");
+
+ MLIRContext *ctx = getContext();
+ ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+ IntegerType i32Ty = b.getI32Type();
+ IntegerType i64Ty = b.getI64Type();
+
+ static constexpr int regBits = 32;
+ auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
+ auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
+ if (!srcType || srcType.getRank() != 1 || !dstType ||
+ dstType.getRank() != 1)
+ return rewriter.notifyMatchFailure(
+ op, "expected 1-D vector; canonicalize pattern handles other shapes");
+
+ auto srcElemType = srcType.getElementType();
+ auto dstElemType = dstType.getElementType();
+ int srcBW = srcType.getElementTypeBitWidth();
+ int dstBW = dstType.getElementTypeBitWidth();
+ int numElems = srcType.getNumElements();
+
+ auto packedKind = op.getPackedKind();
+ NVVM::FPRoundingModeAttr rndModeAttr = op.getRndAttr();
+ NVVM::SaturationModeAttr satModeAttr = op.getSatAttr();
+ auto reluBoolAttr = op.getReluAttr();
+ Value randomBits = adaptor.getRandomBits();
+ Type actualDstFloatType =
+ getActualDstFloatType(ctx, dstElemType, packedKind);
+
+ // STEP 1: bitcast input vector to i32 register vector.
+ // f64 source: decompose to f64->f32 (LLVM fptrunc), then lower f32->dst.
+ Value input = adaptor.getIn();
+ if (srcBW == 64) {
+ auto f32VecTy = VectorType::get(srcType.getShape(), b.getF32Type());
+ input = b.create<LLVM::FPTruncOp>(f32VecTy, input);
+ srcType = f32VecTy;
+ srcElemType = b.getF32Type();
+ srcBW = 32;
+ if (dstBW == 32) {
+ rewriter.replaceOp(op, input);
+ return success();
+ }
+ }
+
+ int srcI32Elems = numElems * srcBW / regBits;
+ int dstI32Elems = numElems * dstBW / regBits;
+ Value srcI32Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(srcI32Elems, i32Ty), input);
+ Value dstI32Vec =
+ b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
+
+ // STEP 2: look up the conversion op from the (srcType, dstType) table.
+ auto convEntry = lookupTruncConvOp(srcElemType, dstElemType);
+ if (!convEntry)
+ return rewriter.notifyMatchFailure(
+ op, "unsupported type combination for truncation");
+ auto [convOp, srcStepDecrement] = *convEntry;
+
+ // STEP 3: pack conversion results into destination i32 vector.
+ const int srcStep = srcBW / dstBW;
+ const int resultBW =
+ dstBW * 2; // each conversion produces 2 (packed) elements
+ const int numConvsPerI32 = regBits / resultBW;
+
+ for (int srcIdx = 0, dstIdx = 0; dstIdx < dstI32Elems;
+ srcIdx += srcStep, dstIdx++) {
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ Value dstValue;
+
+ if (numConvsPerI32 == 1) {
+ // f16/bf16 destinations
+ dstValue = createTruncConversion(
+ b, ctx, convOp, srcI32Vec, srcIdx, rndModeAttr, satModeAttr,
+ reluBoolAttr, dstElemType, actualDstFloatType, randomBits);
+ } else {
+ // f8/f6 destinations: pack sub-results via vector insert + bitcast.
+ auto subResultType = IntegerType::get(ctx, resultBW);
+ auto subVecTy = VectorType::get(numConvsPerI32, subResultType);
+ Value subVec = b.create<LLVM::UndefOp>(subVecTy);
+
+ int insertIdx = numConvsPerI32 - 1;
+ int curStep = srcStep;
+ while (curStep > 0) {
+ curStep -= srcStepDecrement;
+ Value subResult = createTruncConversion(
+ b, ctx, convOp, srcI32Vec, srcIdx + curStep, rndModeAttr,
+ satModeAttr, reluBoolAttr, dstElemType, actualDstFloatType,
+ /*randomBits=*/Value());
+ subVec = b.create<LLVM::InsertElementOp>(
+ subVec, subResult,
+ b.create<LLVM::ConstantOp>(i64Ty,
+ b.getI64IntegerAttr(insertIdx)));
+ insertIdx--;
+ }
+
+ dstValue = b.create<LLVM::BitcastOp>(i32Ty, subVec);
+ }
+
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ }
+
+ Type convertedType = getTypeConverter()->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ auto dstVec = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
+ rewriter.replaceOp(op, dstVec);
+ return success();
+ }
+};
+
+//===----------------------------------------------------------------------===//
+// NVGPUFPExtOpLowering
+//===----------------------------------------------------------------------===//
+
+/// Extension conversion op identifier for nvgpu.convert.fpext lowering.
+enum class ExtConvOp {
+ F8x2_TO_F16x2,
+ F8x2_TO_BF16x2,
+ F6x2_TO_F16x2,
+ F4x2_TO_F16x2,
+};
+
+enum class ExtSrcKind { F8, F6, F4 };
+enum class ExtDstKind { F16, BF16 };
+
+struct ExtTableEntry {
+ ExtSrcKind src;
+ ExtDstKind dst;
+ ExtConvOp convOp;
+};
+
+static constexpr ExtTableEntry kExtTable[] = {
+ {ExtSrcKind::F8, ExtDstKind::F16, ExtConvOp::F8x2_TO_F16x2},
+ {ExtSrcKind::F8, ExtDstKind::BF16, ExtConvOp::F8x2_TO_BF16x2},
+ {ExtSrcKind::F6, ExtDstKind::F16, ExtConvOp::F6x2_TO_F16x2},
+ {ExtSrcKind::F4, ExtDstKind::F16, ExtConvOp::F4x2_TO_F16x2},
+};
+
+static bool isExtConvertibleF8Type(Type t) {
+ return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(t);
+}
+
+static std::optional<ExtSrcKind> classifyExtSrcType(Type t) {
+ if (isExtConvertibleF8Type(t))
+ return ExtSrcKind::F8;
+ int bitWidth = t.getIntOrFloatBitWidth();
+ if (isa<IntegerType>(t) && bitWidth == 8)
+ return ExtSrcKind::F6;
+ if (bitWidth == 4)
+ return ExtSrcKind::F4;
+ return std::nullopt;
+}
+
+static std::optional<ExtDstKind> classifyExtDstType(Type t) {
+ if (t.isF16())
+ return ExtDstKind::F16;
+ if (t.isBF16())
+ return ExtDstKind::BF16;
+ return std::nullopt;
+}
+
+static std::optional<ExtConvOp> lookupExtConvOp(Type srcElemType,
+ Type dstElemType) {
+ auto srcKind = classifyExtSrcType(srcElemType);
+ auto dstKind = classifyExtDstType(dstElemType);
+ if (!srcKind || !dstKind)
+ return std::nullopt;
+ for (const auto &entry : kExtTable) {
+ if (entry.src == *srcKind && entry.dst == *dstKind)
+ return entry.convOp;
+ }
+ return std::nullopt;
+}
+
+static Type getActualSrcFloatType(MLIRContext *ctx, Type elemType,
+ nvgpu::SubBytesPackedKind packedKind) {
+ if (isa<IntegerType>(elemType)) {
+ if (packedKind == nvgpu::SubBytesPackedKind::U6_UNPACK_U8_E3M2)
+ return Float6E3M2FNType::get(ctx);
+ return Float6E2M3FNType::get(ctx);
+ }
+ return elemType;
+}
+
+/// Create a typed NVVM extension conversion.
+/// For f8/f6: src is vector<2xi8>. For f4: src is i8.
+/// Returns i32 (bitcast from vector<2xf16> or vector<2xbf16>).
+static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
+ ExtConvOp convOp, Value src, BoolAttr reluAttr,
+ Type actualSrcFloatType,
+ Value extScaleFactor = Value()) {
+ IntegerType i32Ty = b.getI32Type();
+ auto srcTyAttr = TypeAttr::get(actualSrcFloatType);
+
+ switch (convOp) {
+ case ExtConvOp::F8x2_TO_F16x2: {
+ Value r = NVVM::ConvertF8x2ToF16x2Op::create(
+ b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case ExtConvOp::F8x2_TO_BF16x2: {
+ Value r = NVVM::ConvertF8x2ToBF16x2Op::create(
+ b, VectorType::get(2, b.getBF16Type()), src, srcTyAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case ExtConvOp::F6x2_TO_F16x2: {
+ Value r = NVVM::ConvertF6x2ToF16x2Op::create(
+ b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case ExtConvOp::F4x2_TO_F16x2: {
+ Value r = NVVM::ConvertF4x2ToF16x2Op::create(
+ b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ }
+ llvm_unreachable("unhandled ExtConvOp");
+}
+
+struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
+ using ConvertOpToLLVMPattern<nvgpu::FPExtOp>::ConvertOpToLLVMPattern;
+
+ LogicalResult
+ matchAndRewrite(nvgpu::FPExtOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ // Tensor inputs are not handled here; they must be converted to vectors
+ // by other means before this pattern runs.
+ if (isa<RankedTensorType>(op.getIn().getType()))
+ return rewriter.notifyMatchFailure(
+ op, "tensor inputs not handled; type converter should lower first");
+
+ MLIRContext *ctx = getContext();
+ ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+ IntegerType i8Ty = b.getI8Type();
+ IntegerType i16Ty = b.getI16Type();
+ IntegerType i32Ty = b.getI32Type();
+ IntegerType i64Ty = b.getI64Type();
+
+ static constexpr int regBits = 32;
+ auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
+ auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
+ if (!srcType || srcType.getRank() != 1 || !dstType ||
+ dstType.getRank() != 1)
+ return rewriter.notifyMatchFailure(
+ op, "expected 1-D vector; canonicalize pattern handles other shapes");
+
+ auto srcElemType = srcType.getElementType();
+ auto dstElemType = dstType.getElementType();
+ int srcBW = srcType.getElementTypeBitWidth();
+ int dstBW = dstType.getElementTypeBitWidth();
+ int numElems = srcType.getNumElements();
+
+ auto packedKind = op.getPackedKind();
+ auto reluBoolAttr = op.getReluAttr();
+ Type actualSrcFloatType =
+ getActualSrcFloatType(ctx, srcElemType, packedKind);
+
+ assert(dstBW == 16 || dstBW == 32 || dstBW == 64);
+
+ // Wide source (f16/bf16/f32) to wide destination (f32/f64): single FPExt.
+ if (srcBW >= 16 && dstBW >= 32) {
+ Value result = adaptor.getIn();
+ if (srcElemType != dstElemType) {
+ Type convertedType = getTypeConverter()->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ result = b.create<LLVM::FPExtOp>(convertedType, result);
+ }
+ rewriter.replaceOp(op, result);
+ return success();
+ }
+
+ // Narrow source (f8/f6/f4): NVVM typed op produces f16/bf16; optionally
+ // followed by FPExt to the final f32/f64 destination.
+ bool needsFinalFPExt = (dstBW >= 32);
+ // e8m0 must go through bf16 (only available NVVM op); others use f16.
+ Type intermediateDstElem = dstElemType;
+ if (needsFinalFPExt && llvm::isa<Float8E8M0FNUType>(srcElemType))
+ intermediateDstElem = b.getBF16Type();
+ else if (needsFinalFPExt)
+ intermediateDstElem = b.getF16Type();
+ int intermediateDstBW = needsFinalFPExt ? 16 : dstBW;
+
+ // STEP 1: bitcast input vector to i32 register vector.
+ int srcI32Elems = numElems * srcBW / regBits;
+ int dstI32Elems = numElems * intermediateDstBW / regBits;
+ Value srcI32Vec = b.create<LLVM::BitcastOp>(
+ VectorType::get(srcI32Elems, i32Ty), adaptor.getIn());
+ Value dstI32Vec =
+ b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
+
+ // STEP 2: look up the conversion op from the (srcType, dstType) table.
+ auto convOpOpt = lookupExtConvOp(srcElemType, intermediateDstElem);
+ if (!convOpOpt)
+ return rewriter.notifyMatchFailure(
+ op, "unsupported type combination for extension");
+ ExtConvOp convOp = *convOpOpt;
+ Value extScaleFactor;
+
+ // STEP 3: iterate over source i32 elements, producing destination i32s.
+ for (int srcIdx = 0, dstIdx = 0; srcIdx < srcI32Elems; srcIdx++) {
+ Value srcI32 = b.create<LLVM::ExtractElementOp>(
+ srcI32Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(srcIdx)));
+
+ if (srcBW == 8) {
+ // f8/f6: one i32 holds 4 bytes -> split into 2 pairs of i16 -> 2 convs.
+ Value i16Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(2, i16Ty), srcI32);
+ for (int half = 0; half < 2; half++) {
+ Value halfI16 = b.create<LLVM::ExtractElementOp>(
+ i16Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(half)));
+ Value src =
+ b.create<LLVM::BitcastOp>(VectorType::get(2, i8Ty), halfI16);
+ Value dstValue =
+ createExtConversion(b, ctx, convOp, src, reluBoolAttr,
+ actualSrcFloatType, extScaleFactor);
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ dstIdx++;
+ }
+ } else {
+ // f4: one i32 holds 4 bytes -> each byte is one conversion input.
+ Value i8Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(4, i8Ty), srcI32);
+ for (int byteIdx = 0; byteIdx < 4; byteIdx++) {
+ Value src = b.create<LLVM::ExtractElementOp>(
+ i8Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(byteIdx)));
+ Value dstValue =
+ createExtConversion(b, ctx, convOp, src, reluBoolAttr,
+ actualSrcFloatType, extScaleFactor);
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ dstIdx++;
+ }
+ }
+ }
+
+ // STEP 4: produce final result.
+ Type convertedType = getTypeConverter()->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ Value result;
+ if (needsFinalFPExt) {
+ auto intermediateVecTy = VectorType::get(numElems, intermediateDstElem);
+ Value intermediateVec =
+ b.create<LLVM::BitcastOp>(intermediateVecTy, dstI32Vec);
+ result = b.create<LLVM::FPExtOp>(convertedType, intermediateVec);
+ } else {
+ result = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
+ }
+ rewriter.replaceOp(op, result);
+ return success();
+ }
+};
+
+static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
+ int step) {
+ static constexpr int regBits = 32;
+ auto ceilDiv = [](int64_t x, int64_t y) { return (x + y - 1) / y; };
+ int64_t padded =
+ std::max(ceilDiv(numElems * srcBW, regBits) * regBits / srcBW,
+ ceilDiv(numElems * dstBW, regBits) * regBits / dstBW);
+ return ceilDiv(padded, step) * step;
+}
+
+/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext: handles
+/// scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors. Runs as
+/// an OpRewritePattern on MLIR types before LLVM type conversion.
+template <typename CvtOp, bool IsTrunc>
+struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
+ using OpRewritePattern<CvtOp>::OpRewritePattern;
+
+ LogicalResult matchAndRewrite(CvtOp op,
+ PatternRewriter &rewriter) const override {
+ Type inType = op.getIn().getType();
+ Type outType = op.getOut().getType();
+
+ // Tensor inputs are not handled here; they must be converted to vectors
+ // by other means before this pattern runs.
+ if (isa<RankedTensorType>(inType))
+ return failure();
+
+ Type srcElemTy = getElementTypeOrSelf(inType);
+ Type dstElemTy = getElementTypeOrSelf(outType);
+ int srcBW = srcElemTy.getIntOrFloatBitWidth();
+ int dstBW = dstElemTy.getIntOrFloatBitWidth();
+
+ bool isScalar = !isa<VectorType>(inType);
+ auto srcVecTy = dyn_cast<VectorType>(inType);
+ bool isMultiRank = srcVecTy && srcVecTy.getRank() > 1;
+ int64_t numElems = isScalar ? 1 : srcVecTy.getNumElements();
+ int step = IsTrunc ? srcBW / dstBW : dstBW / srcBW;
+ int64_t paddedElems = computePaddedElems(numElems, srcBW, dstBW, step);
+ bool needsPad = (paddedElems != numElems);
+
+ if (!isScalar && !isMultiRank && !needsPad)
+ return failure();
+
+ ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+ Value input = op.getIn();
+
+ if (isScalar)
+ input = vector::BroadcastOp::create(b, VectorType::get({1}, srcElemTy),
+ input);
+
+ if (isMultiRank)
+ input = vector::ShapeCastOp::create(
+ b, VectorType::get({numElems}, srcElemTy), input);
+
+ if (needsPad) {
+ auto paddedTy = VectorType::get({paddedElems}, srcElemTy);
+ Value zero = arith::ConstantOp::create(
+ b, DenseElementsAttr::get(paddedTy, b.getZeroAttr(srcElemTy)));
+ input = vector::InsertStridedSliceOp::create(
+ b, input, zero, SmallVector<int64_t>{0}, SmallVector<int64_t>{1});
+ }
+
+ auto cvtDstTy =
+ VectorType::get({needsPad ? paddedElems : numElems}, dstElemTy);
+ Value cvt;
+ if constexpr (IsTrunc)
+ cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(),
+ op.getPackedKindAttr(), op.getSatAttr(),
+ op.getReluAttr(), op.getRandomBits());
+ else
+ cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(),
+ op.getPackedKindAttr(), op.getReluAttr());
+ Value result = cvt;
+
+ if (needsPad)
+ result = vector::ExtractStridedSliceOp::create(
+ b, result, SmallVector<int64_t>{0}, SmallVector<int64_t>{numElems},
+ SmallVector<int64_t>{1});
+
+ if (isMultiRank)
+ result =
+ vector::ShapeCastOp::create(b, cast<VectorType>(outType), result);
+
+ if (isScalar)
+ result = vector::ExtractOp::create(b, result, SmallVector<int64_t>{0});
+
+ rewriter.replaceOp(op, result);
+ return success();
+ }
+};
+
+using NVGPUFPTruncCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::FPTruncOp, true>;
+using NVGPUFPExtCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::FPExtOp, false>;
} // namespace
void mlir::nvgpu::populateCommonGPUTypeAndAttributeConversions(
@@ -1753,7 +2418,12 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUWarpgroupMmaOpLowering, // nvgpu.warpgroup.mma
NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
+ NVGPUFPTruncOpLowering, // nvgpu.convert.fptrunc
+ NVGPUFPExtOpLowering, // nvgpu.convert.fpext
MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
+
+ patterns.add<NVGPUFPTruncCanonicalizePattern, NVGPUFPExtCanonicalizePattern>(
+ patterns.getContext());
}
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 2b2686a9163bf..5e3b7c8df515e 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -700,6 +700,162 @@ LogicalResult RcpOp::verify() {
return success();
}
+//===----------------------------------------------------------------------===//
+// NVGPU_CvtFPTruncOp
+//===----------------------------------------------------------------------===//
+
+static bool isShapedContainerType(Type t) {
+ return llvm::isa<VectorType, RankedTensorType, UnrankedTensorType>(t);
+}
+
+static LogicalResult verifyConversionShapes(Operation *op, Type inType,
+ Type outType) {
+ if (llvm::isa<UnrankedTensorType>(inType) ||
+ llvm::isa<UnrankedTensorType>(outType))
+ return op->emitOpError("unranked tensor types are not supported, got ")
+ << inType << " and " << outType;
+ bool srcIsShaped = isShapedContainerType(inType);
+ bool dstIsShaped = isShapedContainerType(outType);
+ if (srcIsShaped != dstIsShaped)
+ return op->emitOpError("input and output must both be scalars or both be "
+ "vectors/tensors, got ")
+ << inType << " and " << outType;
+ if (srcIsShaped) {
+ auto srcShaped = llvm::cast<ShapedType>(inType);
+ auto dstShaped = llvm::cast<ShapedType>(outType);
+ if (srcShaped.getRank() == 0 || dstShaped.getRank() == 0)
+ return op->emitOpError("rank-0 shaped types are not supported, use "
+ "scalar type instead");
+ if (srcShaped.getShape() != dstShaped.getShape())
+ return op->emitOpError("input and output shapes must match, got ")
+ << inType << " and " << outType;
+ if (llvm::isa<VectorType>(inType) != llvm::isa<VectorType>(outType))
+ return op->emitOpError("input and output must be the same container "
+ "type (both vector or both tensor), got ")
+ << inType << " and " << outType;
+ }
+ return success();
+}
+
+LogicalResult FPTruncOp::verify() {
+ Type inType = getIn().getType();
+ Type outType = getType();
+ Type srcType = getElementTypeOrSelf(inType);
+ Type dstType = getElementTypeOrSelf(outType);
+ SubBytesPackedKind packedKind = getPackedKind();
+ int srcBitWidth = srcType.getIntOrFloatBitWidth();
+ int dstBitWidth = dstType.getIntOrFloatBitWidth();
+ auto rnd = getRnd();
+
+ if (auto result = verifyConversionShapes(getOperation(), inType, outType);
+ failed(result))
+ return result;
+
+ if (srcBitWidth <= dstBitWidth)
+ return emitOpError("result type ")
+ << dstType << " must be narrower than operand type " << srcType;
+
+ if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
+ return emitOpError("input type must be 64/32/16 bitwidth, but got ")
+ << srcBitWidth;
+
+ if (dstBitWidth == 6)
+ return emitOpError("currently doesn't support fp6 compact result type");
+
+ if ((packedKind == SubBytesPackedKind::U6_UNPACK_U8_E3M2 ||
+ packedKind == SubBytesPackedKind::U6_UNPACK_U8_E2M3) &&
+ !dstType.isInteger(8))
+ return emitOpError("result type expects `i8` with `u6_unpack_u8` packed "
+ "kind, but got ")
+ << dstType;
+
+ if (packedKind == SubBytesPackedKind::COMPACT &&
+ !llvm::isa<FloatType>(dstType))
+ return emitOpError("result type expects float type with `compact` packed "
+ "kind, but got ")
+ << dstType;
+
+ if (llvm::isa<Float8E8M0FNUType>(dstType)) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
+ rnd != mlir::NVVM::FPRoundingMode::RP)
+ return emitOpError("expects RZ or RP rounding mode when result type is "
+ "e8m0, but got ")
+ << getRndAttr();
+ } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
+ // TODO: Currently, we only support conversions which fit into a single i32
+ // register. Support f32->f8/f6/f4 conversions with RS rounding.
+ if (!(srcBitWidth == 32 && (dstBitWidth == 16)))
+ return emitOpError("RS (stochastic) rounding is only supported for "
+ "f32->f16/bf16, got ")
+ << srcType << " -> " << dstType;
+ if (!getRandomBits())
+ return emitOpError("random_bits operand is required with RS rounding");
+ } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
+ return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
+ }
+
+ if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
+ return emitOpError("random_bits can only be used with RS rounding mode");
+
+ return success();
+}
+
+//===----------------------------------------------------------------------===//
+// NVGPU_CvtFPExtOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult FPExtOp::verify() {
+ Type inType = getIn().getType();
+ Type outType = getType();
+ Type srcType = getElementTypeOrSelf(inType);
+ Type dstType = getElementTypeOrSelf(outType);
+ SubBytesPackedKind packedKind = getPackedKind();
+ int srcBitWidth = srcType.getIntOrFloatBitWidth();
+ int dstBitWidth = dstType.getIntOrFloatBitWidth();
+ auto rnd = getRnd();
+
+ if (auto result = verifyConversionShapes(getOperation(), inType, outType);
+ failed(result))
+ return result;
+
+ if (srcBitWidth >= dstBitWidth)
+ return emitOpError("result type ")
+ << dstType << " must be wider than operand type " << srcType;
+
+ if (dstBitWidth != 16 && dstBitWidth != 32 && dstBitWidth != 64)
+ return emitOpError("result type must be 16, 32, or 64 bitwidth, but got ")
+ << dstBitWidth;
+
+ if (srcBitWidth == 6)
+ return emitOpError("currently doesn't support fp6 compact input type");
+
+ if ((packedKind == SubBytesPackedKind::U6_UNPACK_U8_E3M2 ||
+ packedKind == SubBytesPackedKind::U6_UNPACK_U8_E2M3) &&
+ !srcType.isInteger(8))
+ return emitOpError("input type expects `i8` with `u6_unpack_u8` packed "
+ "kind, but got ")
+ << srcType;
+
+ if (packedKind == SubBytesPackedKind::COMPACT &&
+ !llvm::isa<FloatType>(srcType))
+ return emitOpError("input type expects float type with `compact` packed "
+ "kind, but got ")
+ << srcType;
+
+ if (llvm::isa<Float8E8M0FNUType>(srcType) &&
+ !llvm::isa<BFloat16Type>(dstType) && !dstType.isF32())
+ return emitOpError("expects bf16 or f32 output type when input type is "
+ "e8m0.");
+
+ if (rnd != mlir::NVVM::FPRoundingMode::RN)
+ return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
+
+ if (getRelu() && llvm::isa<BFloat16Type>(dstType))
+ return emitOpError("relu is not supported for bf16 destination");
+
+ return success();
+}
+
//===----------------------------------------------------------------------===//
// TableGen'd dialect, type, and op definitions
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
new file mode 100644
index 0000000000000..08407fe4bebf1
--- /dev/null
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
@@ -0,0 +1,466 @@
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm -convert-vector-to-llvm | FileCheck %s --check-prefix=CHECK-E2E
+
+// Basic aligned vector extension to f16/bf16.
+
+// CHECK-LABEL: @cvt_float_e4m3fn_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf8E4M3FN>
+func.func @cvt_float_e4m3fn_to_f16(%in : vector<8xf8E4M3FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf8E4M3FN> to vector<8xi8>
+ // CHECK: llvm.bitcast %[[CAST]] : vector<8xi8> to vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<4xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f8E4M3FN)
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f8E4M3FN)
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e5m2_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf8E5M2>
+func.func @cvt_float_e5m2_to_f16(%in : vector<8xf8E5M2>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf8E5M2> to vector<8xi8>
+ // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[CAST]] : vector<8xi8> to vector<2xi32>
+ // CHECK: %[[OUT_I32:.+]] = llvm.mlir.undef : vector<4xi32>
+ // CHECK: llvm.extractelement %[[IN_I32]]
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: llvm.extractelement {{.*}} : vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f8E5M2)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xf16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<4xi32>
+ // CHECK: llvm.extractelement {{.*}} : vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f8E5M2)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xf16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<4xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E5M2> to vector<8xf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e8m0_to_bf16(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf8E8M0FNU>
+func.func @cvt_float_e8m0_to_bf16(%in : vector<8xf8E8M0FNU>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf8E8M0FNU> to vector<8xi8>
+ // CHECK: llvm.bitcast %[[CAST]] : vector<8xi8> to vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<4xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK-SAME: : vector<2xi8>(f8E8M0FNU)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xbf16> to i32
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK-SAME: : vector<2xi8>(f8E8M0FNU)
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E8M0FNU> to vector<8xbf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e2m3_to_f16(
+// CHECK: %[[IN:.+]]: vector<8xi8>
+func.func @cvt_float_e2m3_to_f16(%in : vector<8xi8>) {
+ // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[IN]] : vector<8xi8> to vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<4xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f6E2M3FN)
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f6E2M3FN)
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<8xi8> to vector<8xf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e3m2_to_f16(
+// CHECK: %[[IN:.+]]: vector<8xi8>
+func.func @cvt_float_e3m2_to_f16(%in : vector<8xi8>) {
+ // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[IN]] : vector<8xi8> to vector<2xi32>
+ // CHECK: %[[OUT_I32:.+]] = llvm.mlir.undef : vector<4xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f6E3M2FN)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xf16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<4xi32>
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK-SAME: : vector<2xi8>(f6E3M2FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<4xi32>
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<8xi8> to vector<8xf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e2m1_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<16xf4E2M1FN>
+func.func @cvt_float_e2m1_to_f16(%in : vector<16xf4E2M1FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<16xf4E2M1FN> to vector<16xi4>
+ // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[CAST]] : vector<16xi4> to vector<2xi32>
+ // CHECK: %[[OUT_I32:.+]] = llvm.mlir.undef : vector<8xi32>
+ // CHECK: llvm.extractelement %[[IN_I32]]
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<4xi8>
+ // CHECK: llvm.extractelement {{.*}} : vector<4xi8>
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK-SAME: : i8(f4E2M1FN)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xf16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<8xi32>
+ // CHECK: llvm.extractelement {{.*}} : vector<4xi8>
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK-SAME: : i8(f4E2M1FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<8xi32>
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.insertelement {{.*}} : vector<8xi32>
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.insertelement {{.*}} : vector<8xi32>
+ // CHECK: llvm.bitcast {{.*}} : i32 to vector<4xi8>
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xf16>
+ %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf16>
+ return
+}
+
+// Extension to f32 (two-step: narrow -> f16/bf16 -> f32).
+
+// -----
+
+// CHECK-LABEL: @fpext_e4m3fn_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf8E4M3FN>
+func.func @fpext_e4m3fn_to_f32(%in : vector<4xf8E4M3FN>) -> vector<4xf32> {
+ // CHECK: builtin.unrealized_conversion_cast %[[IN]] : vector<4xf8E4M3FN> to vector<4xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi8> to vector<1xi32>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
+ // CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_e5m2_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf8E5M2>
+func.func @fpext_e5m2_to_f32(%in : vector<4xf8E5M2>) -> vector<4xf32> {
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
+ // CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_e8m0_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf8E8M0FNU>
+func.func @fpext_e8m0_to_f32(%in : vector<4xf8E8M0FNU>) -> vector<4xf32> {
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK: nvvm.convert.f8x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
+ // CHECK: llvm.fpext {{.*}} : vector<4xbf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E8M0FNU> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_e2m3_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xi8>
+func.func @fpext_e2m3_to_f32(%in : vector<4xi8>) -> vector<4xf32> {
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
+ // CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<4xi8> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_e2m1_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf4E2M1FN>
+func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ // CHECK: llvm.fpext {{.*}} : vector<8xf16> to vector<8xf32>
+ %out = nvgpu.convert.fpext %in : vector<8xf4E2M1FN> to vector<8xf32>
+ return %out : vector<8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_f16_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf16>
+func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
+ // CHECK: llvm.fpext %[[IN]] : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf16> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_bf16_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<4xbf16>
+func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
+ // CHECK: llvm.fpext %[[IN]] : vector<4xbf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xbf16> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// Extension to f64.
+
+// CHECK-LABEL: @fpext_f16_to_f64
+func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
+ // CHECK: llvm.fpext %{{.*}} : vector<4xf16> to vector<4xf64>
+ // CHECK-NOT: llvm.fpext
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf16> to vector<4xf64>
+ return %out : vector<4xf64>
+}
+
+// CHECK-LABEL: @fpext_bf16_to_f64
+func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
+ // CHECK: llvm.fpext %{{.*}} : vector<4xbf16> to vector<4xf64>
+ // CHECK-NOT: llvm.fpext
+ %out = nvgpu.convert.fpext %arg0 : vector<4xbf16> to vector<4xf64>
+ return %out : vector<4xf64>
+}
+
+// CHECK-LABEL: @fpext_f32_to_f64
+func.func @fpext_f32_to_f64(%arg0: vector<4xf32>) -> vector<4xf64> {
+ // CHECK-NOT: nvvm
+ // CHECK: llvm.fpext %{{.*}} : vector<4xf32> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf32> to vector<4xf64>
+ return %out : vector<4xf64>
+}
+
+// CHECK-LABEL: @fpext_f8_to_f64
+func.func @fpext_f8_to_f64(%arg0: vector<4xf8E4M3FN>) -> vector<4xf64> {
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.fpext {{.*}} to {{.*}}f64
+ // CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
+ return %out : vector<4xf64>
+}
+
+// CHECK-LABEL: @fpext_f4_to_f64
+func.func @fpext_f4_to_f64(%arg0: vector<8xf4E2M1FN>) -> vector<8xf64> {
+ // CHECK: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.fpext {{.*}} to {{.*}}f64
+ // CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
+ %out = nvgpu.convert.fpext %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
+ return %out : vector<8xf64>
+}
+
+// CHECK-LABEL: @fpext_e2m3_to_f64
+func.func @fpext_e2m3_to_f64(%arg0: vector<4xi8>) -> vector<4xf64> {
+ // CHECK: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.fpext {{.*}} to {{.*}}f64
+ // CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
+ %out = nvgpu.convert.fpext %arg0 {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<4xi8> to vector<4xf64>
+ return %out : vector<4xf64>
+}
+
+// Scalar inputs (canonicalize: broadcast + pad + extract).
+
+// CHECK-LABEL: @fpext_scalar_f8_to_f16
+// CHECK-SAME: %[[IN:.+]]: f8E4M3FN
+func.func @fpext_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
+ // CHECK: vector.broadcast %[[IN]] : f8E4M3FN to vector<1xf8E4M3FN>
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ // CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ return %out : f16
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_scalar_f8_to_f32
+// CHECK-SAME: %[[IN:.+]]: f8E4M3FN
+func.func @fpext_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
+ // CHECK: vector.broadcast
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.fpext
+ // CHECK: vector.extract_strided_slice
+ // CHECK: vector.extract {{.*}}[0] : f32 from vector<1xf32>
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ return %out : f32
+}
+
+// Multi-rank vectors (canonicalize: shape_cast flatten).
+
+// CHECK-LABEL: @fpext_v2x4_f8_to_f16
+// CHECK-SAME: %[[IN:.+]]: vector<2x4xf8E4M3FN>
+func.func @fpext_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
+ // CHECK: vector.shape_cast %[[IN]] : vector<2x4xf8E4M3FN> to vector<8xf8E4M3FN>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ return %out : vector<2x4xf16>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_v2x4_f8_to_f32
+// CHECK-SAME: %[[IN:.+]]: vector<2x4xf8E4M3FN>
+func.func @fpext_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
+ // CHECK: vector.shape_cast {{.*}} to vector<8xf8E4M3FN>
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.fpext
+ // CHECK: vector.shape_cast {{.*}} to vector<2x4xf32>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ return %out : vector<2x4xf32>
+}
+
+// Non-aligned 1-D vectors (canonicalize: pad via insert/extract_strided_slice).
+
+// CHECK-LABEL: @fpext_v3f8_to_v3f16
+// CHECK-SAME: %[[IN:.+]]: vector<3xf8E5M2>
+func.func @fpext_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
+ // CHECK: vector.insert_strided_slice %[[IN]]
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ return %out : vector<3xf16>
+}
+
+// -----
+
+// CHECK-LABEL: @fpext_v3_f8_to_f32
+// CHECK-SAME: %[[IN:.+]]: vector<3xf8E5M2>
+func.func @fpext_v3_f8_to_f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.fpext
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ return %out : vector<3xf32>
+}
+
+// Multi-rank + padding combined.
+
+// CHECK-LABEL: @fpext_v3x1_f8_to_f16
+// CHECK-SAME: %[[IN:.+]]: vector<3x1xf8E4M3FN>
+func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
+ // CHECK: vector.shape_cast %[[IN]] : vector<3x1xf8E4M3FN> to vector<3xf8E4M3FN>
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ // CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
+ %out = nvgpu.convert.fpext %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
+ return %out : vector<3x1xf16>
+}
+
+// Relu attribute.
+
+// CHECK-LABEL: @fpext_f8_to_f16_relu
+func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
+ // CHECK: nvvm.convert.f8x2.to.f16x2
+ // CHECK-SAME: relu = true
+ %out = nvgpu.convert.fpext %in {relu = true}
+ : vector<8xf8E4M3FN> to vector<8xf16>
+ return
+}
+
+// End-to-end: no residual vector ops after full lowering.
+
+// CHECK-E2E-LABEL: @e2e_scalar_f8_to_f16
+// CHECK-E2E-NOT: vector.broadcast
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E-NOT: vector.extract
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: return
+func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ return %out : f16
+}
+
+// CHECK-E2E-LABEL: @e2e_v2x4_f8_to_f16
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: return
+func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ return %out : vector<2x4xf16>
+}
+
+// CHECK-E2E-LABEL: @e2e_v3f8_to_v3f16
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: return
+func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ return %out : vector<3xf16>
+}
+
+// CHECK-E2E-LABEL: @e2e_scalar_f8_to_f32
+// CHECK-E2E-NOT: vector.broadcast
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E-NOT: vector.extract
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: llvm.fpext
+// CHECK-E2E: return
+func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ return %out : f32
+}
+
+// CHECK-E2E-LABEL: @e2e_v2x4_f8_to_f32
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: llvm.fpext
+// CHECK-E2E: return
+func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ return %out : vector<2x4xf32>
+}
+
+// CHECK-E2E-LABEL: @e2e_v3f8_to_v3f32
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
+// CHECK-E2E: llvm.fpext
+// CHECK-E2E: return
+func.func @e2e_v3f8_to_v3f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ return %out : vector<3xf32>
+}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
new file mode 100644
index 0000000000000..e8a5c27f22917
--- /dev/null
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
@@ -0,0 +1,327 @@
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm -convert-vector-to-llvm | FileCheck %s --check-prefix=CHECK-E2E
+
+// Basic aligned vector inputs.
+
+// CHECK-LABEL: @cvt_float_f32_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf32>
+func.func @cvt_float_f32_to_f16(%in : vector<4xf32>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<4xf32> to vector<4xi32>
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
+ // CHECK-SAME: : vector<2xf16>
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
+ // CHECK-SAME: : vector<2xf16>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_f32_to_f16_v8(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
+func.func @cvt_float_f32_to_f16_v8(%in : vector<8xf32>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<8xf32> to vector<8xi32>
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf16>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_f32_to_bf16(
+// CHECK-SAME: %[[IN:.+]]: vector<4xf32>
+func.func @cvt_float_f32_to_bf16(%in : vector<4xf32>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<4xf32> to vector<4xi32>
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
+ // CHECK-SAME: : vector<2xbf16>
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xbf16>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_f32_to_e4m3(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
+func.func @cvt_float_f32_to_e4m3(%in : vector<8xf32>) {
+ // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[IN]] : vector<8xf32> to vector<8xi32>
+ // CHECK: %[[OUT_I32:.+]] = llvm.mlir.undef : vector<2xi32>
+ // CHECK: %[[IDX_0:.+]] = llvm.mlir.constant(0 : i64) : i64
+ // CHECK: %[[SUB_VEC_0:.+]] = llvm.mlir.undef : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
+ // CHECK-SAME: : i16(f8E4M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: : i16(f8E4M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi32>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_f32_to_e2m3(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
+func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<8xf32> to vector<8xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f6x2
+ // CHECK-SAME: : i16(f6E2M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f6x2
+ // CHECK-SAME: : i16(f6E2M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi16>
+ // CHECK: nvvm.convert.f32x2.to.f6x2
+ // CHECK: nvvm.convert.f32x2.to.f6x2
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi32>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
+ %out = nvgpu.convert.fptrunc %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>}: vector<8xf32> to vector<8xi8>
+ return
+}
+
+// Scalar inputs (canonicalize: broadcast + pad + extract).
+
+// CHECK-LABEL: @fptrunc_scalar_f32_to_f16
+// CHECK-SAME: %[[IN:.+]]: f32
+func.func @fptrunc_scalar_f32_to_f16(%in : f32) -> f16 {
+ // CHECK: vector.broadcast %[[IN]] : f32 to vector<1xf32>
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ // CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
+ %out = nvgpu.convert.fptrunc %in : f32 to f16
+ return %out : f16
+}
+
+// CHECK-LABEL: @fptrunc_scalar_f32_to_bf16
+// CHECK-SAME: %[[IN:.+]]: f32
+func.func @fptrunc_scalar_f32_to_bf16(%in : f32) -> bf16 {
+ // CHECK: vector.broadcast %[[IN]]
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK: vector.extract
+ %out = nvgpu.convert.fptrunc %in : f32 to bf16
+ return %out : bf16
+}
+
+// CHECK-LABEL: @fptrunc_scalar_f64_to_f32
+func.func @fptrunc_scalar_f64_to_f32(%arg0: f64) -> f32 {
+ // CHECK: vector.broadcast
+ // CHECK: llvm.fptrunc
+ // CHECK: vector.extract
+ %out = nvgpu.convert.fptrunc %arg0 : f64 to f32
+ return %out : f32
+}
+
+// Multi-rank vectors (canonicalize: shape_cast flatten).
+
+// CHECK-LABEL: @fptrunc_v2x4_f32_to_f16
+// CHECK-SAME: %[[IN:.+]]: vector<2x4xf32>
+func.func @fptrunc_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
+ // CHECK: vector.shape_cast %[[IN]] : vector<2x4xf32> to vector<8xf32>
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ return %out : vector<2x4xf16>
+}
+
+// CHECK-LABEL: @fptrunc_v4x2_f32_to_f8
+// CHECK-SAME: %[[IN:.+]]: vector<4x2xf32>
+func.func @fptrunc_v4x2_f32_to_f8(%in : vector<4x2xf32>) -> vector<4x2xf8E4M3FN> {
+ // CHECK: vector.shape_cast %[[IN]] : vector<4x2xf32> to vector<8xf32>
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK: vector.shape_cast {{.*}} to vector<4x2xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
+ return %out : vector<4x2xf8E4M3FN>
+}
+
+// Non-aligned 1-D vectors (canonicalize: pad via insert/extract_strided_slice).
+
+// CHECK-LABEL: @fptrunc_v1f32_to_v1f16
+// CHECK-SAME: %[[IN:.+]]: vector<1xf32>
+func.func @fptrunc_v1f32_to_v1f16(%in : vector<1xf32>) -> vector<1xf16> {
+ // CHECK: vector.insert_strided_slice %[[IN]]
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fptrunc %in : vector<1xf32> to vector<1xf16>
+ return %out : vector<1xf16>
+}
+
+// CHECK-LABEL: @fptrunc_v3f16_to_v3f8
+// CHECK-SAME: %[[IN:.+]]: vector<3xf16>
+func.func @fptrunc_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
+ // CHECK: vector.insert_strided_slice %[[IN]]
+ // CHECK: nvvm.convert.f16x2.to.f8x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ return %out : vector<3xf8E4M3FN>
+}
+
+// Multi-rank + padding combined.
+
+// CHECK-LABEL: @fptrunc_v3x1_f32_to_f16
+// CHECK-SAME: %[[IN:.+]]: vector<3x1xf32>
+func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
+ // CHECK: vector.shape_cast %[[IN]] : vector<3x1xf32> to vector<3xf32>
+ // CHECK: vector.insert_strided_slice
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ // CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<3x1xf32> to vector<3x1xf16>
+ return %out : vector<3x1xf16>
+}
+
+// f64 source truncation.
+
+// CHECK-LABEL: @fptrunc_f64_to_f32
+func.func @fptrunc_f64_to_f32(%arg0: vector<4xf64>) -> vector<4xf32> {
+ // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
+ // CHECK-NOT: nvvm
+ %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf32>
+ return %out : vector<4xf32>
+}
+
+// CHECK-LABEL: @fptrunc_f64_to_f16
+func.func @fptrunc_f64_to_f16(%arg0: vector<2xf64>) -> vector<2xf16> {
+ // CHECK: vector.insert_strided_slice
+ // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xf16>
+ return %out : vector<2xf16>
+}
+
+// CHECK-LABEL: @fptrunc_f64_to_bf16
+func.func @fptrunc_f64_to_bf16(%arg0: vector<2xf64>) -> vector<2xbf16> {
+ // CHECK: vector.insert_strided_slice
+ // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xbf16>
+ return %out : vector<2xbf16>
+}
+
+// CHECK-LABEL: @fptrunc_f64_to_f8
+func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
+ // CHECK: vector.insert_strided_slice
+ // CHECK: llvm.fptrunc %{{.*}} : vector<8xf64> to vector<8xf32>
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK: vector.extract_strided_slice
+ %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
+ return %out : vector<4xf8E4M3FN>
+}
+
+// Saturation and relu attributes.
+
+// CHECK-LABEL: @fptrunc_f32_to_f8_satfinite
+func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
+ %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<satfinite>}
+ : vector<8xf32> to vector<8xf8E4M3FN>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_f16_relu
+func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: relu = true
+ %out = nvgpu.convert.fptrunc %in {relu = true}
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// Default SATFINITE behavior.
+
+// CHECK-LABEL: @fptrunc_f32_to_f8_default_sat
+func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_f16_default_sat
+func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_f16_explicit_none
+func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-NOT: satfinite
+ %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<none>}
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// Stochastic rounding (RS + random_bits).
+
+// CHECK-LABEL: @fptrunc_f32_to_f16_rs
+func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
+ %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_bf16_rs
+func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
+ %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ : vector<4xf32> to vector<4xbf16>
+ return
+}
+
+// End-to-end: no residual vector ops after full lowering.
+
+// CHECK-E2E-LABEL: @e2e_scalar_f32_to_f16
+// CHECK-E2E-NOT: vector.broadcast
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E-NOT: vector.extract
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
+// CHECK-E2E: return
+func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
+ %out = nvgpu.convert.fptrunc %in : f32 to f16
+ return %out : f16
+}
+
+// CHECK-E2E-LABEL: @e2e_v2x4_f32_to_f16
+// CHECK-E2E-NOT: vector.shape_cast
+// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
+// CHECK-E2E: return
+func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
+ %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ return %out : vector<2x4xf16>
+}
+
+// CHECK-E2E-LABEL: @e2e_v3f16_to_v3f8
+// CHECK-E2E-NOT: vector.insert_strided_slice
+// CHECK-E2E-NOT: vector.extract_strided_slice
+// CHECK-E2E: nvvm.convert.f16x2.to.f8x2
+// CHECK-E2E: return
+func.func @e2e_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
+ %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ return %out : vector<3xf8E4M3FN>
+}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
new file mode 100644
index 0000000000000..a2f8fd6ec48f8
--- /dev/null
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
@@ -0,0 +1,106 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+
+// -----
+
+func.func @fpext_wider(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op result type 'f4E2M1FN' must be wider than operand type 'f8E5M2'}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ return
+}
+
+// -----
+
+func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op result type must be 16, 32, or 64 bitwidth, but got 8}}
+ %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fpext_fp6_compact(%in : vector<16xf6E2M3FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op currently doesn't support fp6 compact input type}}
+ %out = nvgpu.convert.fpext %in : vector<16xf6E2M3FN> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_u6_packed_not_i8(%in : vector<16xf8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input type expects `i8` with `u6_unpack_u8` packed kind, but got 'f8E4M3FN'}}
+ %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<16xf8E4M3FN> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_compact_not_float(%in : vector<16xi8>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input type expects float type with `compact` packed kind, but got 'i8'}}
+ %out = nvgpu.convert.fpext %in : vector<16xi8> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op expects bf16 or f32 output type when input type is e8m0.}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E8M0FNU> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.fpext %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<16xf8E5M2> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op relu is not supported for bf16 destination}}
+ %out = nvgpu.convert.fpext %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
+ return
+}
+
+// -----
+
+func.func @fpext_shape_mismatch(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output shapes must match}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<8xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_scalar_vector_mismatch(%in : f8E4M3FN) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to vector<1xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_rank0_tensor(%in : tensor<f8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.fpext %in : tensor<f8E4M3FN> to tensor<f16>
+ return
+}
+
+// -----
+
+func.func @fpext_container_mismatch(%in : vector<4xf8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to tensor<4xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_unranked_tensor(%in : tensor<*xf8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.fpext %in : tensor<*xf8E4M3FN> to tensor<*xf16>
+ return
+}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
new file mode 100644
index 0000000000000..9e295b4ad25a0
--- /dev/null
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
@@ -0,0 +1,126 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+
+// -----
+
+func.func @fptrunc_narrower(%in : vector<16xf16>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type 'f32' must be narrower than operand type 'f16'}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf16> to vector<16xf32>
+ return
+}
+
+// -----
+
+func.func @fptrunc_src_bitwidth(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input type must be 64/32/16 bitwidth, but got 8}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_fp6_compact(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op currently doesn't support fp6 compact result type}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<16xf6E2M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_u6_packed_not_i8(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type expects `i8` with `u6_unpack_u8` packed kind, but got 'f8E4M3FN'}}
+ %out = nvgpu.convert.fptrunc %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<16xf32> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_compact_not_float(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type expects float type with `compact` packed kind, but got 'i8'}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<16xi8>
+ return
+}
+
+// -----
+
+func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rn>}
+ : vector<16xf32> to vector<16xf8E8M0FNU>
+ return
+}
+
+// -----
+
+func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ : vector<16xf32> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits operand is required with RS rounding}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// -----
+
+func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ : vector<16xf32> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits can only be used with RS rounding mode}}
+ %out = nvgpu.convert.fptrunc %in, %rbits
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// -----
+
+func.func @fptrunc_shape_mismatch(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output shapes must match}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<8xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_scalar_vector_mismatch(%in : f32) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.fptrunc %in : f32 to vector<1xf16>
+ return
+}
+
+// -----
+
+func.func @fptrunc_rank0_tensor(%in : tensor<f32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.fptrunc %in : tensor<f32> to tensor<f16>
+ return
+}
+
+// -----
+
+func.func @fptrunc_container_mismatch(%in : vector<4xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to tensor<4xf16>
+ return
+}
+
+// -----
+
+func.func @fptrunc_unranked_tensor(%in : tensor<*xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.fptrunc %in : tensor<*xf32> to tensor<*xf16>
+ return
+}
>From 1cff6f6c7644ebb0467cffb4a64b11928874c8a4 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 26 May 2026 14:49:27 +0000
Subject: [PATCH 02/11] fix formatting
---
mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 8b992b67aed29..08cbefee015c2 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -2289,9 +2289,9 @@ static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
return ceilDiv(padded, step) * step;
}
-/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext: handles
-/// scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors. Runs as
-/// an OpRewritePattern on MLIR types before LLVM type conversion.
+/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext:
+/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
+/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
template <typename CvtOp, bool IsTrunc>
struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
using OpRewritePattern<CvtOp>::OpRewritePattern;
>From cae401fd4789b033cb9e96c2d5ba3514f476dc02 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 27 May 2026 13:37:59 +0000
Subject: [PATCH 03/11] address comments and refactor
---
mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 13 -
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 21 +-
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 345 +++++++++---------
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 34 --
.../NVGPUToNVVM/nvgpu-convert-fpext.mlir | 33 +-
.../NVGPUToNVVM/nvgpu-convert-fptrunc.mlir | 36 +-
.../NVGPU/nvgpu-convert-fpext-invalid.mlir | 24 --
.../NVGPU/nvgpu-convert-fptrunc-invalid.mlir | 24 --
8 files changed, 225 insertions(+), 305 deletions(-)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index e56aca779cd1e..38c8bb5c9aa2c 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -101,19 +101,6 @@ def TensorMapInterleaveKind : I32EnumAttr<"TensorMapInterleaveKind",
let cppNamespace = "::mlir::nvgpu";
}
-def SubBytesPackedCompact : I32EnumAttrCase<"COMPACT", 0, "compact">;
-def SubBytesPackedU6UnpackU8E3M2 : I32EnumAttrCase<"U6_UNPACK_U8_E3M2", 1, "u6_unpack_u8_e3m2">;
-def SubBytesPackedU6UnpackU8E2M3 : I32EnumAttrCase<"U6_UNPACK_U8_E2M3", 2, "u6_unpack_u8_e2m3">;
-def SubBytesPackedKind : I32EnumAttr<"SubBytesPackedKind",
- "Sub-bytes packed kind type",
- [SubBytesPackedCompact, SubBytesPackedU6UnpackU8E3M2,
- SubBytesPackedU6UnpackU8E2M3]> {
- let genSpecializedAttr = 0;
- let cppNamespace = "::mlir::nvgpu";
-}
-def SubBytesPackedKindAttr : EnumAttr<NVGPU_Dialect, SubBytesPackedKind, "subbytes_packedkind"> {
- let assemblyFormat = "`<` $value `>`";
-}
def RcpApprox : I32EnumAttrCase<"APPROX", 0, "approx">;
def RcpRN : I32EnumAttrCase<"RN", 1, "rn">;
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 07408ac18f95b..4ffca888c0260 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -675,11 +675,6 @@ def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
// NVGPU Conversion Ops
//===----------------------------------------------------------------------===//
-def Int8OrFloatLike : TypeConstraint<
- Or<[FloatLike.predicate,
- I8.predicate,
- ValueSemanticsContainerOf<[I8]>.predicate]>,
- "scalar, vector, or tensor of i8 or floats">;
def AnyI32Like : TypeOrValueSemanticsContainer<I32, "scalar i32 or vector of i32">;
def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
@@ -690,30 +685,26 @@ def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
Destination must be strictly narrower than source.
Supported paths: f32->f16, f32->bf16, f32->f8, f32->f6, f32->f4,
- f16->f8, f16->f4, bf16->f8, bf16->f4.
+ f16->f8, f16->f6, f16->f4, bf16->f8, bf16->f6, bf16->f4.
The `random_bits` operand enables stochastic rounding (RS mode)
for f32->f16/bf16 conversions. When provided, `rnd` must be RS.
- For f6 types (f6E3M2FN/f6E2M3FN), result is i8 with 2-bit MSB
- padding; use `packed_kind` to specify the f6 variant. `compact` is not
- a supported `packed_kind` for f6 types.
-
Example:
```mlir
%r = nvgpu.cvt_fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %r = nvgpu.cvt_fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
%r = nvgpu.cvt_fptrunc %in : f32 to f16
%r = nvgpu.cvt_fptrunc %in : vector<2x4xf16> to vector<2x4xf8E5M2>
```
}];
let arguments = (ins FloatLike:$in,
DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
- DefaultValuedAttr<SubBytesPackedKindAttr, "SubBytesPackedKind::COMPACT">:$packed_kind,
DefaultValuedAttr<SaturationModeAttr, "NVVM::SaturationMode::SATFINITE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
Optional<I32>:$random_bits
);
- let results = (outs Int8OrFloatLike:$out);
+ let results = (outs FloatLike:$out);
let assemblyFormat = "$in (`,` $random_bits^)? attr-dict `:` type($in) `to` type($out)";
let hasVerifier = 1;
}
@@ -729,18 +720,16 @@ def NVGPU_FPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
intermediate, then LLVM FPExt). e8m0->bf16 and e8m0->f32 supported
(f32 goes through bf16 intermediate).
- For f6 types, input is i8 with `packed_kind` specifying the f6 variant.
-
Example:
```mlir
%r = nvgpu.cvt_fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %r = nvgpu.cvt_fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
%r = nvgpu.cvt_fpext %in : vector<4xf8E5M2> to vector<4xf32>
%r = nvgpu.cvt_fpext %in : f8E4M3FN to f32
```
}];
- let arguments = (ins Int8OrFloatLike:$in,
+ let arguments = (ins FloatLike:$in,
DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
- DefaultValuedAttr<SubBytesPackedKindAttr, "SubBytesPackedKind::COMPACT">:$packed_kind,
DefaultValuedAttr<BoolAttr, "false">:$relu);
let results = (outs FloatLike:$out);
let assemblyFormat = "$in attr-dict `:` type($in) `to` type($out)";
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 08cbefee015c2..99ab025a6a27e 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1713,90 +1713,99 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
};
//===----------------------------------------------------------------------===//
-// FPTruncOp Lowering
+// NVGPUFPTruncOp Lowering
//===----------------------------------------------------------------------===//
+enum class FPKind { F32, BF16, F16, F8, F6, F4 };
+
+static int getEffectiveBitWidth(int bitWidth) {
+ // f6 types are 6-bit but NVVM Ops expect 8-bit (i8) containers.
+ return bitWidth == 6 ? 8 : bitWidth;
+}
+
+static std::optional<FPKind> classifyFPType(Type t) {
+ static constexpr auto isConvertibleF8Type = [](Type t) {
+ return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(t);
+ };
+ static constexpr auto isConvertibleF6Type = [](Type t) {
+ return isa<Float6E2M3FNType, Float6E3M2FNType>(t);
+ };
+ static constexpr auto isConvertibleF4Type = [](Type t) {
+ return isa<Float4E2M1FNType>(t);
+ };
+
+ if (t.isF32())
+ return FPKind::F32;
+ if (t.isBF16())
+ return FPKind::BF16;
+ if (t.isF16())
+ return FPKind::F16;
+ if (isConvertibleF8Type(t))
+ return FPKind::F8;
+ if (isConvertibleF6Type(t))
+ return FPKind::F6;
+ if (isConvertibleF4Type(t))
+ return FPKind::F4;
+ return std::nullopt;
+}
+
+/// Number of source-side i32 register slots consumed by each NVVM convert Op.
+static int getNumSrcI32PerConv(FPKind src) {
+ return src == FPKind::F32 ? 2 : 1;
+}
+
/// Conversion op identifier for nvgpu.convert.fptrunc lowering dispatch table.
-enum class TruncConvOp {
+enum class FPTruncConvOp {
F32x2_TO_F16x2,
F32x2_TO_BF16x2,
F32x2_TO_F8x2,
F32x2_TO_F6x2,
F32x2_TO_F4x2,
F16x2_TO_F8x2,
+ F16x2_TO_F6x2,
F16x2_TO_F4x2,
BF16x2_TO_F8x2,
+ BF16x2_TO_F6x2,
BF16x2_TO_F4x2,
};
-enum class TruncSrcKind { F32, F16, BF16 };
-
-enum class TruncDstKind { F16, BF16, F8, F6, F4 };
-
-struct TruncTableEntry {
- TruncSrcKind src;
- TruncDstKind dst;
- TruncConvOp convOp;
- int srcStepDecrement; // 2 for f32 pairs, 1 for f16x2/bf16x2
+struct FPTruncTableEntry {
+ FPKind src;
+ FPKind dst;
+ FPTruncConvOp convOp;
};
-static constexpr TruncTableEntry kTruncTable[] = {
+static constexpr FPTruncTableEntry kFPTruncTable[] = {
// f32 source
- {TruncSrcKind::F32, TruncDstKind::F16, TruncConvOp::F32x2_TO_F16x2, 2},
- {TruncSrcKind::F32, TruncDstKind::BF16, TruncConvOp::F32x2_TO_BF16x2, 2},
- {TruncSrcKind::F32, TruncDstKind::F8, TruncConvOp::F32x2_TO_F8x2, 2},
- {TruncSrcKind::F32, TruncDstKind::F6, TruncConvOp::F32x2_TO_F6x2, 2},
- {TruncSrcKind::F32, TruncDstKind::F4, TruncConvOp::F32x2_TO_F4x2, 2},
+ {FPKind::F32, FPKind::F16, FPTruncConvOp::F32x2_TO_F16x2},
+ {FPKind::F32, FPKind::BF16, FPTruncConvOp::F32x2_TO_BF16x2},
+ {FPKind::F32, FPKind::F8, FPTruncConvOp::F32x2_TO_F8x2},
+ {FPKind::F32, FPKind::F6, FPTruncConvOp::F32x2_TO_F6x2},
+ {FPKind::F32, FPKind::F4, FPTruncConvOp::F32x2_TO_F4x2},
// f16 source
- {TruncSrcKind::F16, TruncDstKind::F8, TruncConvOp::F16x2_TO_F8x2, 1},
- {TruncSrcKind::F16, TruncDstKind::F4, TruncConvOp::F16x2_TO_F4x2, 1},
+ {FPKind::F16, FPKind::F8, FPTruncConvOp::F16x2_TO_F8x2},
+ {FPKind::F16, FPKind::F6, FPTruncConvOp::F16x2_TO_F6x2},
+ {FPKind::F16, FPKind::F4, FPTruncConvOp::F16x2_TO_F4x2},
// bf16 source
- {TruncSrcKind::BF16, TruncDstKind::F8, TruncConvOp::BF16x2_TO_F8x2, 1},
- {TruncSrcKind::BF16, TruncDstKind::F4, TruncConvOp::BF16x2_TO_F4x2, 1},
+ {FPKind::BF16, FPKind::F8, FPTruncConvOp::BF16x2_TO_F8x2},
+ {FPKind::BF16, FPKind::F6, FPTruncConvOp::BF16x2_TO_F6x2},
+ {FPKind::BF16, FPKind::F4, FPTruncConvOp::BF16x2_TO_F4x2},
};
-static bool isConvertibleF8Type(Type t) {
- return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(t);
-}
-
-static std::optional<TruncSrcKind> classifySrcType(Type t) {
- if (t.isF32())
- return TruncSrcKind::F32;
- if (t.isF16())
- return TruncSrcKind::F16;
- if (t.isBF16())
- return TruncSrcKind::BF16;
- return std::nullopt;
-}
-
-static std::optional<TruncDstKind> classifyDstType(Type t) {
- if (t.isF16())
- return TruncDstKind::F16;
- if (t.isBF16())
- return TruncDstKind::BF16;
- if (isConvertibleF8Type(t))
- return TruncDstKind::F8;
- int bitWidth = t.getIntOrFloatBitWidth();
- if (isa<IntegerType>(t) && bitWidth == 8)
- return TruncDstKind::F6;
- if (bitWidth == 4)
- return TruncDstKind::F4;
- return std::nullopt;
-}
-
-static std::optional<std::pair<TruncConvOp, int>>
-lookupTruncConvOp(Type srcElemType, Type dstElemType) {
- auto srcKind = classifySrcType(srcElemType);
- auto dstKind = classifyDstType(dstElemType);
+static std::optional<FPTruncTableEntry> lookupTruncConvOp(Type srcElemType,
+ Type dstElemType) {
+ auto srcKind = classifyFPType(srcElemType);
+ auto dstKind = classifyFPType(dstElemType);
if (!srcKind || !dstKind)
return std::nullopt;
- for (const auto &entry : kTruncTable) {
+ for (const auto &entry : kFPTruncTable) {
if (entry.src == *srcKind && entry.dst == *dstKind)
- return {{entry.convOp, entry.srcStepDecrement}};
+ return entry;
}
return std::nullopt;
}
+/// Extract a single element from a vector.
static Value extractElement(ImplicitLocOpBuilder &b, Value srcVec, int idx) {
IntegerType i64Ty = b.getI64Type();
return b.create<LLVM::ExtractElementOp>(
@@ -1804,7 +1813,6 @@ static Value extractElement(ImplicitLocOpBuilder &b, Value srcVec, int idx) {
}
/// Extract a pair of f32 values from an i32 vector at the given base index.
-/// Returns {f32_lo (lower index), f32_hi (higher index)}.
static std::pair<Value, Value> extractF32Pair(ImplicitLocOpBuilder &b,
Value srcI32Vec, int baseIdx) {
FloatType f32Ty = b.getF32Type();
@@ -1814,18 +1822,16 @@ static std::pair<Value, Value> extractF32Pair(ImplicitLocOpBuilder &b,
b.create<LLVM::BitcastOp>(f32Ty, elem1)};
}
+/// Extract a vector of elements of size i32 from an i32 vector and bitcast to
+/// the specified vector type.
static Value extractAndBitcast(ImplicitLocOpBuilder &b, Value srcI32Vec,
int idx, VectorType vecTy) {
Value elem = extractElement(b, srcI32Vec, idx);
return b.create<LLVM::BitcastOp>(vecTy, elem);
}
-/// Dispatch to the specific NVVM conversion op based on TruncConvOp,
-/// extract source operands, and return the native result.
-/// - f16/bf16 destinations: returns i32 (bitcast from vector<2xf16/bf16>)
-/// - f8/f6 destinations: returns i16 (packed 2 x f8/f6)
-/// - f4 destinations: returns i8 (packed 2 x f4)
-/// Create a sub-byte conversion from an f32 pair source.
+/// Create a sub-byte conversion from an f32 pair source and return the native
+/// result.
template <typename ConvertOp, typename... Args>
static Value convertFromF32Pair(ImplicitLocOpBuilder &b, Value srcI32Vec,
int srcBaseIdx, Type resultTy, Args &&...args) {
@@ -1833,7 +1839,8 @@ static Value convertFromF32Pair(ImplicitLocOpBuilder &b, Value srcI32Vec,
return b.create<ConvertOp>(resultTy, hi, lo, std::forward<Args>(args)...);
}
-/// Create a sub-byte conversion from a packed f16x2/bf16x2 source.
+/// Create a sub-byte conversion from a packed f16x2/bf16x2 source and return
+/// the native result.
template <typename ConvertOp, typename... Args>
static Value convertFromPacked(ImplicitLocOpBuilder &b, Value srcI32Vec,
int srcBaseIdx, Type srcElemTy, Type resultTy,
@@ -1843,8 +1850,9 @@ static Value convertFromPacked(ImplicitLocOpBuilder &b, Value srcI32Vec,
return b.create<ConvertOp>(resultTy, src, std::forward<Args>(args)...);
}
+/// Create a typed NVVM truncation conversion.
static Value createTruncConversion(
- ImplicitLocOpBuilder &b, MLIRContext *ctx, TruncConvOp convOp,
+ ImplicitLocOpBuilder &b, MLIRContext *ctx, FPTruncConvOp convOp,
Value srcI32Vec, int srcBaseIdx, NVVM::FPRoundingModeAttr rndAttr,
NVVM::SaturationModeAttr satAttr, BoolAttr reluAttr, Type dstElemType,
Type actualDstFloatType, Value randomBits = Value()) {
@@ -1855,67 +1863,63 @@ static Value createTruncConversion(
auto actualDstTyAttr = TypeAttr::get(actualDstFloatType);
switch (convOp) {
- case TruncConvOp::F32x2_TO_F16x2: {
+ case FPTruncConvOp::F32x2_TO_F16x2: {
auto [lo, hi] = extractF32Pair(b, srcI32Vec, srcBaseIdx);
Value r = b.create<NVVM::ConvertF32x2ToF16x2Op>(
VectorType::get(2, b.getF16Type()), hi, lo, randomBits, rndAttr,
satAttr, reluAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
- case TruncConvOp::F32x2_TO_BF16x2: {
+ case FPTruncConvOp::F32x2_TO_BF16x2: {
auto [lo, hi] = extractF32Pair(b, srcI32Vec, srcBaseIdx);
Value r = b.create<NVVM::ConvertF32x2ToBF16x2Op>(
VectorType::get(2, b.getBF16Type()), hi, lo, randomBits, rndAttr,
satAttr, reluAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
- case TruncConvOp::F32x2_TO_F8x2:
+ case FPTruncConvOp::F32x2_TO_F8x2:
return convertFromF32Pair<NVVM::ConvertF32x2ToF8x2Op>(
b, srcI32Vec, srcBaseIdx, i16Ty, rndAttr, satAttr, reluAttr, dstTyAttr);
- case TruncConvOp::F32x2_TO_F6x2:
+ case FPTruncConvOp::F32x2_TO_F6x2:
return convertFromF32Pair<NVVM::ConvertF32x2ToF6x2Op>(
b, srcI32Vec, srcBaseIdx, i16Ty, reluAttr, actualDstTyAttr);
- case TruncConvOp::F32x2_TO_F4x2:
+ case FPTruncConvOp::F32x2_TO_F4x2:
return convertFromF32Pair<NVVM::ConvertF32x2ToF4x2Op>(
b, srcI32Vec, srcBaseIdx, i8Ty, reluAttr, dstTyAttr);
- case TruncConvOp::F16x2_TO_F8x2:
+ case FPTruncConvOp::F16x2_TO_F8x2:
return convertFromPacked<NVVM::ConvertF16x2ToF8x2Op>(
b, srcI32Vec, srcBaseIdx, b.getF16Type(), i16Ty, reluAttr, dstTyAttr);
- case TruncConvOp::F16x2_TO_F4x2:
+ case FPTruncConvOp::F16x2_TO_F6x2:
+ return convertFromPacked<NVVM::ConvertF16x2ToF6x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getF16Type(), i16Ty, reluAttr,
+ actualDstTyAttr);
+ case FPTruncConvOp::F16x2_TO_F4x2:
return convertFromPacked<NVVM::ConvertF16x2ToF4x2Op>(
b, srcI32Vec, srcBaseIdx, b.getF16Type(), i8Ty, reluAttr,
actualDstTyAttr);
- case TruncConvOp::BF16x2_TO_F8x2:
+ case FPTruncConvOp::BF16x2_TO_F8x2:
return convertFromPacked<NVVM::ConvertBF16x2ToF8x2Op>(
b, srcI32Vec, srcBaseIdx, b.getBF16Type(), i16Ty, rndAttr, satAttr,
reluAttr, dstTyAttr);
- case TruncConvOp::BF16x2_TO_F4x2:
+ case FPTruncConvOp::BF16x2_TO_F6x2:
+ return convertFromPacked<NVVM::ConvertBF16x2ToF6x2Op>(
+ b, srcI32Vec, srcBaseIdx, b.getBF16Type(), i16Ty, reluAttr,
+ actualDstTyAttr);
+ case FPTruncConvOp::BF16x2_TO_F4x2:
return convertFromPacked<NVVM::ConvertBF16x2ToF4x2Op>(
b, srcI32Vec, srcBaseIdx, b.getBF16Type(), i8Ty, reluAttr,
actualDstTyAttr);
}
- llvm_unreachable("unhandled TruncConvOp");
+ llvm_unreachable("unhandled FPTruncConvOp");
}
struct NVGPUFPTruncOpLowering
: public ConvertOpToLLVMPattern<nvgpu::FPTruncOp> {
using ConvertOpToLLVMPattern<nvgpu::FPTruncOp>::ConvertOpToLLVMPattern;
- static Type getActualDstFloatType(MLIRContext *ctx, Type elemType,
- nvgpu::SubBytesPackedKind packedKind) {
- if (isa<IntegerType>(elemType)) {
- if (packedKind == nvgpu::SubBytesPackedKind::U6_UNPACK_U8_E3M2)
- return Float6E3M2FNType::get(ctx);
- return Float6E2M3FNType::get(ctx);
- }
- return elemType;
- }
-
LogicalResult
matchAndRewrite(nvgpu::FPTruncOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- // Tensor inputs are not handled here; they must be converted to vectors
- // by other means before this pattern runs.
if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
op, "tensor inputs not handled; type converter should lower first");
@@ -1924,8 +1928,8 @@ struct NVGPUFPTruncOpLowering
ImplicitLocOpBuilder b(op->getLoc(), rewriter);
IntegerType i32Ty = b.getI32Type();
IntegerType i64Ty = b.getI64Type();
-
static constexpr int regBits = 32;
+
auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
if (!srcType || srcType.getRank() != 1 || !dstType ||
@@ -1939,13 +1943,11 @@ struct NVGPUFPTruncOpLowering
int dstBW = dstType.getElementTypeBitWidth();
int numElems = srcType.getNumElements();
- auto packedKind = op.getPackedKind();
NVVM::FPRoundingModeAttr rndModeAttr = op.getRndAttr();
NVVM::SaturationModeAttr satModeAttr = op.getSatAttr();
auto reluBoolAttr = op.getReluAttr();
Value randomBits = adaptor.getRandomBits();
- Type actualDstFloatType =
- getActualDstFloatType(ctx, dstElemType, packedKind);
+ Type actualDstFloatType = dstElemType;
// STEP 1: bitcast input vector to i32 register vector.
// f64 source: decompose to f64->f32 (LLVM fptrunc), then lower f32->dst.
@@ -1962,8 +1964,11 @@ struct NVGPUFPTruncOpLowering
}
}
+ // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
+ int effectiveDstBW = getEffectiveBitWidth(dstBW);
+
int srcI32Elems = numElems * srcBW / regBits;
- int dstI32Elems = numElems * dstBW / regBits;
+ int dstI32Elems = numElems * effectiveDstBW / regBits;
Value srcI32Vec =
b.create<LLVM::BitcastOp>(VectorType::get(srcI32Elems, i32Ty), input);
Value dstI32Vec =
@@ -1974,12 +1979,13 @@ struct NVGPUFPTruncOpLowering
if (!convEntry)
return rewriter.notifyMatchFailure(
op, "unsupported type combination for truncation");
- auto [convOp, srcStepDecrement] = *convEntry;
+ FPTruncConvOp convOp = convEntry->convOp;
+ int numSrcI32PerConv = getNumSrcI32PerConv(convEntry->src);
// STEP 3: pack conversion results into destination i32 vector.
- const int srcStep = srcBW / dstBW;
+ const int srcStep = srcBW / effectiveDstBW;
const int resultBW =
- dstBW * 2; // each conversion produces 2 (packed) elements
+ effectiveDstBW * 2; // each conversion produces 2 (packed) elements
const int numConvsPerI32 = regBits / resultBW;
for (int srcIdx = 0, dstIdx = 0; dstIdx < dstI32Elems;
@@ -1994,7 +2000,7 @@ struct NVGPUFPTruncOpLowering
b, ctx, convOp, srcI32Vec, srcIdx, rndModeAttr, satModeAttr,
reluBoolAttr, dstElemType, actualDstFloatType, randomBits);
} else {
- // f8/f6 destinations: pack sub-results via vector insert + bitcast.
+ // f8/f6/f4 destinations: pack sub-results via vector insert + bitcast.
auto subResultType = IntegerType::get(ctx, resultBW);
auto subVecTy = VectorType::get(numConvsPerI32, subResultType);
Value subVec = b.create<LLVM::UndefOp>(subVecTy);
@@ -2002,7 +2008,7 @@ struct NVGPUFPTruncOpLowering
int insertIdx = numConvsPerI32 - 1;
int curStep = srcStep;
while (curStep > 0) {
- curStep -= srcStepDecrement;
+ curStep -= numSrcI32PerConv;
Value subResult = createTruncConversion(
b, ctx, convOp, srcI32Vec, srcIdx + curStep, rndModeAttr,
satModeAttr, reluBoolAttr, dstElemType, actualDstFloatType,
@@ -2021,10 +2027,19 @@ struct NVGPUFPTruncOpLowering
b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
}
+ // STEP 4: produce final result.
Type convertedType = getTypeConverter()->convertType(dstType);
assert(convertedType && "failed to convert type");
- auto dstVec = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
- rewriter.replaceOp(op, dstVec);
+ if (convEntry->dst == FPKind::F6) {
+ IntegerType i8Ty = b.getI8Type();
+ auto i8VecTy = VectorType::get(numElems, i8Ty);
+ Value i8Vec = b.create<LLVM::BitcastOp>(i8VecTy, dstI32Vec);
+ Value truncVec = b.create<LLVM::TruncOp>(convertedType, i8Vec);
+ rewriter.replaceOp(op, truncVec);
+ } else {
+ auto dstVec = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
+ rewriter.replaceOp(op, dstVec);
+ }
return success();
}
};
@@ -2034,108 +2049,72 @@ struct NVGPUFPTruncOpLowering
//===----------------------------------------------------------------------===//
/// Extension conversion op identifier for nvgpu.convert.fpext lowering.
-enum class ExtConvOp {
+enum class FPExtConvOp {
F8x2_TO_F16x2,
F8x2_TO_BF16x2,
F6x2_TO_F16x2,
F4x2_TO_F16x2,
};
-enum class ExtSrcKind { F8, F6, F4 };
-enum class ExtDstKind { F16, BF16 };
-
-struct ExtTableEntry {
- ExtSrcKind src;
- ExtDstKind dst;
- ExtConvOp convOp;
+struct FPExtTableEntry {
+ FPKind src;
+ FPKind dst;
+ FPExtConvOp convOp;
};
-static constexpr ExtTableEntry kExtTable[] = {
- {ExtSrcKind::F8, ExtDstKind::F16, ExtConvOp::F8x2_TO_F16x2},
- {ExtSrcKind::F8, ExtDstKind::BF16, ExtConvOp::F8x2_TO_BF16x2},
- {ExtSrcKind::F6, ExtDstKind::F16, ExtConvOp::F6x2_TO_F16x2},
- {ExtSrcKind::F4, ExtDstKind::F16, ExtConvOp::F4x2_TO_F16x2},
+static constexpr FPExtTableEntry kFPExtTable[] = {
+ {FPKind::F8, FPKind::F16, FPExtConvOp::F8x2_TO_F16x2},
+ {FPKind::F8, FPKind::BF16, FPExtConvOp::F8x2_TO_BF16x2},
+ {FPKind::F6, FPKind::F16, FPExtConvOp::F6x2_TO_F16x2},
+ {FPKind::F4, FPKind::F16, FPExtConvOp::F4x2_TO_F16x2},
};
-static bool isExtConvertibleF8Type(Type t) {
- return isa<Float8E4M3FNType, Float8E5M2Type, Float8E8M0FNUType>(t);
-}
-
-static std::optional<ExtSrcKind> classifyExtSrcType(Type t) {
- if (isExtConvertibleF8Type(t))
- return ExtSrcKind::F8;
- int bitWidth = t.getIntOrFloatBitWidth();
- if (isa<IntegerType>(t) && bitWidth == 8)
- return ExtSrcKind::F6;
- if (bitWidth == 4)
- return ExtSrcKind::F4;
- return std::nullopt;
-}
-
-static std::optional<ExtDstKind> classifyExtDstType(Type t) {
- if (t.isF16())
- return ExtDstKind::F16;
- if (t.isBF16())
- return ExtDstKind::BF16;
- return std::nullopt;
-}
-
-static std::optional<ExtConvOp> lookupExtConvOp(Type srcElemType,
- Type dstElemType) {
- auto srcKind = classifyExtSrcType(srcElemType);
- auto dstKind = classifyExtDstType(dstElemType);
+static std::optional<FPExtTableEntry> lookupExtConvOp(Type srcElemType,
+ Type dstElemType) {
+ auto srcKind = classifyFPType(srcElemType);
+ auto dstKind = classifyFPType(dstElemType);
if (!srcKind || !dstKind)
return std::nullopt;
- for (const auto &entry : kExtTable) {
+ for (const auto &entry : kFPExtTable) {
if (entry.src == *srcKind && entry.dst == *dstKind)
- return entry.convOp;
+ return entry;
}
return std::nullopt;
}
-static Type getActualSrcFloatType(MLIRContext *ctx, Type elemType,
- nvgpu::SubBytesPackedKind packedKind) {
- if (isa<IntegerType>(elemType)) {
- if (packedKind == nvgpu::SubBytesPackedKind::U6_UNPACK_U8_E3M2)
- return Float6E3M2FNType::get(ctx);
- return Float6E2M3FNType::get(ctx);
- }
- return elemType;
-}
-
/// Create a typed NVVM extension conversion.
/// For f8/f6: src is vector<2xi8>. For f4: src is i8.
/// Returns i32 (bitcast from vector<2xf16> or vector<2xbf16>).
static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
- ExtConvOp convOp, Value src, BoolAttr reluAttr,
- Type actualSrcFloatType,
+ FPExtConvOp convOp, Value src,
+ BoolAttr reluAttr, Type actualSrcFloatType,
Value extScaleFactor = Value()) {
IntegerType i32Ty = b.getI32Type();
auto srcTyAttr = TypeAttr::get(actualSrcFloatType);
switch (convOp) {
- case ExtConvOp::F8x2_TO_F16x2: {
+ case FPExtConvOp::F8x2_TO_F16x2: {
Value r = NVVM::ConvertF8x2ToF16x2Op::create(
b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
- case ExtConvOp::F8x2_TO_BF16x2: {
+ case FPExtConvOp::F8x2_TO_BF16x2: {
Value r = NVVM::ConvertF8x2ToBF16x2Op::create(
b, VectorType::get(2, b.getBF16Type()), src, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
- case ExtConvOp::F6x2_TO_F16x2: {
+ case FPExtConvOp::F6x2_TO_F16x2: {
Value r = NVVM::ConvertF6x2ToF16x2Op::create(
b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
- case ExtConvOp::F4x2_TO_F16x2: {
+ case FPExtConvOp::F4x2_TO_F16x2: {
Value r = NVVM::ConvertF4x2ToF16x2Op::create(
b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
}
- llvm_unreachable("unhandled ExtConvOp");
+ llvm_unreachable("unhandled FPExtConvOp");
}
struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
@@ -2144,8 +2123,6 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
LogicalResult
matchAndRewrite(nvgpu::FPExtOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- // Tensor inputs are not handled here; they must be converted to vectors
- // by other means before this pattern runs.
if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
op, "tensor inputs not handled; type converter should lower first");
@@ -2171,10 +2148,8 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
int dstBW = dstType.getElementTypeBitWidth();
int numElems = srcType.getNumElements();
- auto packedKind = op.getPackedKind();
auto reluBoolAttr = op.getReluAttr();
- Type actualSrcFloatType =
- getActualSrcFloatType(ctx, srcElemType, packedKind);
+ Type actualSrcFloatType = srcElemType;
assert(dstBW == 16 || dstBW == 32 || dstBW == 64);
@@ -2193,7 +2168,6 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
// Narrow source (f8/f6/f4): NVVM typed op produces f16/bf16; optionally
// followed by FPExt to the final f32/f64 destination.
bool needsFinalFPExt = (dstBW >= 32);
- // e8m0 must go through bf16 (only available NVVM op); others use f16.
Type intermediateDstElem = dstElemType;
if (needsFinalFPExt && llvm::isa<Float8E8M0FNUType>(srcElemType))
intermediateDstElem = b.getBF16Type();
@@ -2201,20 +2175,30 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
intermediateDstElem = b.getF16Type();
int intermediateDstBW = needsFinalFPExt ? 16 : dstBW;
- // STEP 1: bitcast input vector to i32 register vector.
- int srcI32Elems = numElems * srcBW / regBits;
+ // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
+ int effectiveSrcBW = getEffectiveBitWidth(srcBW);
+
+ // STEP 1: prepare input as i32 register vector.
+ // For f6: zext from vector<Nxi6> to vector<Nxi8>, then bitcast to i32s.
+ Value inputVec = adaptor.getIn();
+ if (srcBW == 6) {
+ auto i8VecTy = VectorType::get(numElems, i8Ty);
+ inputVec = b.create<LLVM::ZExtOp>(i8VecTy, inputVec);
+ }
+
+ int srcI32Elems = numElems * effectiveSrcBW / regBits;
int dstI32Elems = numElems * intermediateDstBW / regBits;
Value srcI32Vec = b.create<LLVM::BitcastOp>(
- VectorType::get(srcI32Elems, i32Ty), adaptor.getIn());
+ VectorType::get(srcI32Elems, i32Ty), inputVec);
Value dstI32Vec =
b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
// STEP 2: look up the conversion op from the (srcType, dstType) table.
- auto convOpOpt = lookupExtConvOp(srcElemType, intermediateDstElem);
- if (!convOpOpt)
+ auto convEntry = lookupExtConvOp(srcElemType, intermediateDstElem);
+ if (!convEntry)
return rewriter.notifyMatchFailure(
op, "unsupported type combination for extension");
- ExtConvOp convOp = *convOpOpt;
+ FPExtConvOp convOp = convEntry->convOp;
Value extScaleFactor;
// STEP 3: iterate over source i32 elements, producing destination i32s.
@@ -2223,7 +2207,7 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
srcI32Vec,
b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(srcIdx)));
- if (srcBW == 8) {
+ if (effectiveSrcBW == 8) {
// f8/f6: one i32 holds 4 bytes -> split into 2 pairs of i16 -> 2 convs.
Value i16Vec =
b.create<LLVM::BitcastOp>(VectorType::get(2, i16Ty), srcI32);
@@ -2282,10 +2266,12 @@ struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
int step) {
static constexpr int regBits = 32;
+ int effSrcBW = getEffectiveBitWidth(srcBW);
+ int effDstBW = getEffectiveBitWidth(dstBW);
auto ceilDiv = [](int64_t x, int64_t y) { return (x + y - 1) / y; };
int64_t padded =
- std::max(ceilDiv(numElems * srcBW, regBits) * regBits / srcBW,
- ceilDiv(numElems * dstBW, regBits) * regBits / dstBW);
+ std::max(ceilDiv(numElems * effSrcBW, regBits) * regBits / effSrcBW,
+ ceilDiv(numElems * effDstBW, regBits) * regBits / effDstBW);
return ceilDiv(padded, step) * step;
}
@@ -2301,8 +2287,6 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
Type inType = op.getIn().getType();
Type outType = op.getOut().getType();
- // Tensor inputs are not handled here; they must be converted to vectors
- // by other means before this pattern runs.
if (isa<RankedTensorType>(inType))
return failure();
@@ -2310,12 +2294,14 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
Type dstElemTy = getElementTypeOrSelf(outType);
int srcBW = srcElemTy.getIntOrFloatBitWidth();
int dstBW = dstElemTy.getIntOrFloatBitWidth();
+ int effSrcBW = getEffectiveBitWidth(srcBW);
+ int effDstBW = getEffectiveBitWidth(dstBW);
bool isScalar = !isa<VectorType>(inType);
auto srcVecTy = dyn_cast<VectorType>(inType);
bool isMultiRank = srcVecTy && srcVecTy.getRank() > 1;
int64_t numElems = isScalar ? 1 : srcVecTy.getNumElements();
- int step = IsTrunc ? srcBW / dstBW : dstBW / srcBW;
+ int step = IsTrunc ? effSrcBW / effDstBW : effDstBW / effSrcBW;
int64_t paddedElems = computePaddedElems(numElems, srcBW, dstBW, step);
bool needsPad = (paddedElems != numElems);
@@ -2345,12 +2331,11 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
VectorType::get({needsPad ? paddedElems : numElems}, dstElemTy);
Value cvt;
if constexpr (IsTrunc)
- cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(),
- op.getPackedKindAttr(), op.getSatAttr(),
+ cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(),
op.getReluAttr(), op.getRandomBits());
else
- cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(),
- op.getPackedKindAttr(), op.getReluAttr());
+ cvt =
+ CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getReluAttr());
Value result = cvt;
if (needsPad)
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 5e3b7c8df515e..7fb68681da4c0 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -742,7 +742,6 @@ LogicalResult FPTruncOp::verify() {
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
Type dstType = getElementTypeOrSelf(outType);
- SubBytesPackedKind packedKind = getPackedKind();
int srcBitWidth = srcType.getIntOrFloatBitWidth();
int dstBitWidth = dstType.getIntOrFloatBitWidth();
auto rnd = getRnd();
@@ -759,22 +758,6 @@ LogicalResult FPTruncOp::verify() {
return emitOpError("input type must be 64/32/16 bitwidth, but got ")
<< srcBitWidth;
- if (dstBitWidth == 6)
- return emitOpError("currently doesn't support fp6 compact result type");
-
- if ((packedKind == SubBytesPackedKind::U6_UNPACK_U8_E3M2 ||
- packedKind == SubBytesPackedKind::U6_UNPACK_U8_E2M3) &&
- !dstType.isInteger(8))
- return emitOpError("result type expects `i8` with `u6_unpack_u8` packed "
- "kind, but got ")
- << dstType;
-
- if (packedKind == SubBytesPackedKind::COMPACT &&
- !llvm::isa<FloatType>(dstType))
- return emitOpError("result type expects float type with `compact` packed "
- "kind, but got ")
- << dstType;
-
if (llvm::isa<Float8E8M0FNUType>(dstType)) {
if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
rnd != mlir::NVVM::FPRoundingMode::RP)
@@ -809,7 +792,6 @@ LogicalResult FPExtOp::verify() {
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
Type dstType = getElementTypeOrSelf(outType);
- SubBytesPackedKind packedKind = getPackedKind();
int srcBitWidth = srcType.getIntOrFloatBitWidth();
int dstBitWidth = dstType.getIntOrFloatBitWidth();
auto rnd = getRnd();
@@ -826,22 +808,6 @@ LogicalResult FPExtOp::verify() {
return emitOpError("result type must be 16, 32, or 64 bitwidth, but got ")
<< dstBitWidth;
- if (srcBitWidth == 6)
- return emitOpError("currently doesn't support fp6 compact input type");
-
- if ((packedKind == SubBytesPackedKind::U6_UNPACK_U8_E3M2 ||
- packedKind == SubBytesPackedKind::U6_UNPACK_U8_E2M3) &&
- !srcType.isInteger(8))
- return emitOpError("input type expects `i8` with `u6_unpack_u8` packed "
- "kind, but got ")
- << srcType;
-
- if (packedKind == SubBytesPackedKind::COMPACT &&
- !llvm::isa<FloatType>(srcType))
- return emitOpError("input type expects float type with `compact` packed "
- "kind, but got ")
- << srcType;
-
if (llvm::isa<Float8E8M0FNUType>(srcType) &&
!llvm::isa<BFloat16Type>(dstType) && !dstType.isF32())
return emitOpError("expects bf16 or f32 output type when input type is "
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
index 08407fe4bebf1..73aaf63641e97 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
@@ -77,9 +77,11 @@ func.func @cvt_float_e8m0_to_bf16(%in : vector<8xf8E8M0FNU>) {
// -----
// CHECK-LABEL: @cvt_float_e2m3_to_f16(
-// CHECK: %[[IN:.+]]: vector<8xi8>
-func.func @cvt_float_e2m3_to_f16(%in : vector<8xi8>) {
- // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[IN]] : vector<8xi8> to vector<2xi32>
+// CHECK-SAME: %[[IN:.+]]: vector<8xf6E2M3FN>
+func.func @cvt_float_e2m3_to_f16(%in : vector<8xf6E2M3FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf6E2M3FN> to vector<8xi6>
+ // CHECK: llvm.zext %[[CAST]] : vector<8xi6> to vector<8xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<8xi8> to vector<2xi32>
// CHECK: llvm.mlir.undef : vector<4xi32>
// CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
// CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
@@ -90,16 +92,18 @@ func.func @cvt_float_e2m3_to_f16(%in : vector<8xi8>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<8xi8> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xf16>
return
}
// -----
// CHECK-LABEL: @cvt_float_e3m2_to_f16(
-// CHECK: %[[IN:.+]]: vector<8xi8>
-func.func @cvt_float_e3m2_to_f16(%in : vector<8xi8>) {
- // CHECK: %[[IN_I32:.+]] = llvm.bitcast %[[IN]] : vector<8xi8> to vector<2xi32>
+// CHECK-SAME: %[[IN:.+]]: vector<8xf6E3M2FN>
+func.func @cvt_float_e3m2_to_f16(%in : vector<8xf6E3M2FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf6E3M2FN> to vector<8xi6>
+ // CHECK: llvm.zext %[[CAST]] : vector<8xi6> to vector<8xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<8xi8> to vector<2xi32>
// CHECK: %[[OUT_I32:.+]] = llvm.mlir.undef : vector<4xi32>
// CHECK: llvm.bitcast {{.*}} : i32 to vector<2xi16>
// CHECK: llvm.bitcast {{.*}} : i16 to vector<2xi8>
@@ -113,7 +117,7 @@ func.func @cvt_float_e3m2_to_f16(%in : vector<8xi8>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<8xi8> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
return
}
@@ -196,13 +200,15 @@ func.func @fpext_e8m0_to_f32(%in : vector<4xf8E8M0FNU>) -> vector<4xf32> {
// -----
// CHECK-LABEL: @fpext_e2m3_to_f32(
-// CHECK-SAME: %[[IN:.+]]: vector<4xi8>
-func.func @fpext_e2m3_to_f32(%in : vector<4xi8>) -> vector<4xf32> {
+// CHECK-SAME: %[[IN:.+]]: vector<4xf6E2M3FN>
+func.func @fpext_e2m3_to_f32(%in : vector<4xf6E2M3FN>) -> vector<4xf32> {
+ // CHECK: builtin.unrealized_conversion_cast %[[IN]] : vector<4xf6E2M3FN> to vector<4xi6>
+ // CHECK: llvm.zext {{.*}} : vector<4xi6> to vector<4xi8>
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<4xi8> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf6E2M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -284,11 +290,12 @@ func.func @fpext_f4_to_f64(%arg0: vector<8xf4E2M1FN>) -> vector<8xf64> {
}
// CHECK-LABEL: @fpext_e2m3_to_f64
-func.func @fpext_e2m3_to_f64(%arg0: vector<4xi8>) -> vector<4xf64> {
+func.func @fpext_e2m3_to_f64(%arg0: vector<4xf6E2M3FN>) -> vector<4xf64> {
+ // CHECK: llvm.zext {{.*}} : vector<8xi6> to vector<8xi8>
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.fpext %arg0 {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>} : vector<4xi8> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
index e8a5c27f22917..bee520a61152a 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
@@ -70,6 +70,39 @@ func.func @cvt_float_f32_to_e4m3(%in : vector<8xf32>) {
return
}
+// CHECK-LABEL: @cvt_float_f16_to_e2m3(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf16>
+func.func @cvt_float_f16_to_e2m3(%in : vector<8xf16>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<8xf16> to vector<4xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi32>
+ // CHECK: llvm.mlir.undef : vector<2xi16>
+ // CHECK: nvvm.convert.f16x2.to.f6x2
+ // CHECK-SAME: : vector<2xf16> -> i16(f6E2M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: nvvm.convert.f16x2.to.f6x2
+ // CHECK-SAME: : vector<2xf16> -> i16(f6E2M3FN)
+ // CHECK: llvm.insertelement {{.*}} : vector<2xi16>
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
+ // CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf16> to vector<8xf6E2M3FN>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_bf16_to_e3m2(
+// CHECK-SAME: %[[IN:.+]]: vector<8xbf16>
+func.func @cvt_float_bf16_to_e3m2(%in : vector<8xbf16>) {
+ // CHECK: llvm.bitcast %[[IN]] : vector<8xbf16> to vector<4xi32>
+ // CHECK: nvvm.convert.bf16x2.to.f6x2
+ // CHECK-SAME: : vector<2xbf16> -> i16(f6E3M2FN)
+ // CHECK: nvvm.convert.bf16x2.to.f6x2
+ // CHECK-SAME: : vector<2xbf16> -> i16(f6E3M2FN)
+ // CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
+ // CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
+ %out = nvgpu.convert.fptrunc %in : vector<8xbf16> to vector<8xf6E3M2FN>
+ return
+}
+
// CHECK-LABEL: @cvt_float_f32_to_e2m3(
// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
@@ -90,7 +123,8 @@ func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
- %out = nvgpu.convert.fptrunc %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e2m3>}: vector<8xf32> to vector<8xi8>
+ // CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
return
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
index a2f8fd6ec48f8..26c3afc5da358 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
@@ -18,30 +18,6 @@ func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
// -----
-func.func @fpext_fp6_compact(%in : vector<16xf6E2M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op currently doesn't support fp6 compact input type}}
- %out = nvgpu.convert.fpext %in : vector<16xf6E2M3FN> to vector<16xf16>
- return
-}
-
-// -----
-
-func.func @fpext_u6_packed_not_i8(%in : vector<16xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input type expects `i8` with `u6_unpack_u8` packed kind, but got 'f8E4M3FN'}}
- %out = nvgpu.convert.fpext %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<16xf8E4M3FN> to vector<16xf16>
- return
-}
-
-// -----
-
-func.func @fpext_compact_not_float(%in : vector<16xi8>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input type expects float type with `compact` packed kind, but got 'i8'}}
- %out = nvgpu.convert.fpext %in : vector<16xi8> to vector<16xf16>
- return
-}
-
-// -----
-
func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
// expected-error @+1 {{'nvgpu.convert.fpext' op expects bf16 or f32 output type when input type is e8m0.}}
%out = nvgpu.convert.fpext %in : vector<16xf8E8M0FNU> to vector<16xf16>
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
index 9e295b4ad25a0..ff899d341c99a 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
@@ -18,30 +18,6 @@ func.func @fptrunc_src_bitwidth(%in : vector<16xf8E5M2>) {
// -----
-func.func @fptrunc_fp6_compact(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op currently doesn't support fp6 compact result type}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<16xf6E2M3FN>
- return
-}
-
-// -----
-
-func.func @fptrunc_u6_packed_not_i8(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type expects `i8` with `u6_unpack_u8` packed kind, but got 'f8E4M3FN'}}
- %out = nvgpu.convert.fptrunc %in {packed_kind = #nvgpu.subbytes_packedkind<u6_unpack_u8_e3m2>} : vector<16xf32> to vector<16xf8E4M3FN>
- return
-}
-
-// -----
-
-func.func @fptrunc_compact_not_float(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type expects float type with `compact` packed kind, but got 'i8'}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<16xi8>
- return
-}
-
-// -----
-
func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
// expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
%out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rn>}
>From 16d21ca1d5b31bed58998ff867d95ee20d20913e Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 27 May 2026 14:06:28 +0000
Subject: [PATCH 04/11] address comments
---
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 19 +++--
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 3 +
.../nvgpu-convert-fpext-large.mlir | 78 +++++++++++++++++++
.../nvgpu-convert-fptrunc-large.mlir | 76 ++++++++++++++++++
4 files changed, 170 insertions(+), 6 deletions(-)
create mode 100644 mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
create mode 100644 mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 4ffca888c0260..14929e6187d54 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -684,8 +684,10 @@ def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
Truncate a floating-point value to a smaller floating-point type.
Destination must be strictly narrower than source.
- Supported paths: f32->f16, f32->bf16, f32->f8, f32->f6, f32->f4,
- f16->f8, f16->f6, f16->f4, bf16->f8, bf16->f6, bf16->f4.
+ Supported paths:
+ f32 -> f16, bf16, f8, f6, f4
+ f16 -> f8, f6, f4
+ bf16 -> f8, f6, f4
The `random_bits` operand enables stochastic rounding (RS mode)
for f32->f16/bf16 conversions. When provided, `rnd` must be RS.
@@ -715,10 +717,15 @@ def NVGPU_FPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
Extend a floating-point value to a wider floating-point type.
Destination must be strictly wider than source.
- Supported paths: f8->f16, f8->bf16, f6->f16, f4->f16, f16->f32,
- bf16->f32. Narrow->f32 uses two-step lowering (NVVM op to f16/bf16
- intermediate, then LLVM FPExt). e8m0->bf16 and e8m0->f32 supported
- (f32 goes through bf16 intermediate).
+ Supported paths:
+ f8 -> f16, bf16
+ f6 -> f16
+ f4 -> f16
+ f16 -> f32
+ bf16 -> f32
+ Narrow->f32 uses two-step lowering (NVVM op to f16/bf16 intermediate,
+ then LLVM FPExt). e8m0->bf16 and e8m0->f32 supported (f32 goes through
+ bf16 intermediate).
Example:
```mlir
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 99ab025a6a27e..46c7783d9f43c 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1807,6 +1807,9 @@ static std::optional<FPTruncTableEntry> lookupTruncConvOp(Type srcElemType,
/// Extract a single element from a vector.
static Value extractElement(ImplicitLocOpBuilder &b, Value srcVec, int idx) {
+ auto vecTy = cast<VectorType>(srcVec.getType());
+ assert(idx >= 0 && idx < vecTy.getNumElements() &&
+ "extractElement: index out of bounds");
IntegerType i64Ty = b.getI64Type();
return b.create<LLVM::ExtractElementOp>(
srcVec, b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(idx)));
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
new file mode 100644
index 0000000000000..35379861f8f5d
--- /dev/null
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
@@ -0,0 +1,78 @@
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
+
+// Large-vector smoke tests for nvgpu.convert.fpext
+
+// CHECK-LABEL: @cvt_large_f8_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<400xf8E4M3FN>
+func.func @cvt_large_f8_to_f16(%in : vector<400xf8E4M3FN>) -> vector<400xf16> {
+ // CHECK: builtin.unrealized_conversion_cast %[[IN]] : vector<400xf8E4M3FN> to vector<400xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<400xi8> to vector<100xi32>
+ // CHECK-COUNT-200: nvvm.convert.f8x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E4M3FN> to vector<400xf16>
+ return %out : vector<400xf16>
+}
+
+// CHECK-LABEL: @cvt_large_e8m0_to_bf16(
+func.func @cvt_large_e8m0_to_bf16(%in : vector<400xf8E8M0FNU>) -> vector<400xbf16> {
+ // CHECK: builtin.unrealized_conversion_cast %{{.*}} : vector<400xf8E8M0FNU> to vector<400xi8>
+ // CHECK-COUNT-200: nvvm.convert.f8x2.to.bf16x2
+ // CHECK-NOT: nvvm.convert.f8x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E8M0FNU> to vector<400xbf16>
+ return %out : vector<400xbf16>
+}
+
+// CHECK-LABEL: @cvt_large_f6_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<400xf6E2M3FN>
+func.func @cvt_large_f6_to_f16(%in : vector<400xf6E2M3FN>) -> vector<400xf16> {
+ // CHECK: builtin.unrealized_conversion_cast %[[IN]] : vector<400xf6E2M3FN> to vector<400xi6>
+ // CHECK: llvm.zext {{.*}} : vector<400xi6> to vector<400xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<400xi8> to vector<100xi32>
+ // CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf16>
+ return %out : vector<400xf16>
+}
+
+// CHECK-LABEL: @cvt_large_f4_to_f16(
+func.func @cvt_large_f4_to_f16(%in : vector<400xf4E2M1FN>) -> vector<400xf16> {
+ // CHECK: builtin.unrealized_conversion_cast %{{.*}} : vector<400xf4E2M1FN> to vector<400xi4>
+ // CHECK: llvm.bitcast {{.*}} : vector<400xi4> to vector<50xi32>
+ // CHECK-COUNT-200: nvvm.convert.f4x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f4x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf4E2M1FN> to vector<400xf16>
+ return %out : vector<400xf16>
+}
+
+// CHECK-LABEL: @cvt_large_f8_to_f32(
+func.func @cvt_large_f8_to_f32(%in : vector<400xf8E5M2>) -> vector<400xf32> {
+ // CHECK-COUNT-200: nvvm.convert.f8x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f8x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
+ // CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E5M2> to vector<400xf32>
+ return %out : vector<400xf32>
+}
+
+// CHECK-LABEL: @cvt_large_f6_to_f32(
+func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
+ // CHECK: llvm.zext {{.*}} : vector<400xi6> to vector<400xi8>
+ // CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f6x2.to.f16x2
+ // CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf32>
+ return %out : vector<400xf32>
+}
+
+// CHECK-LABEL: @cvt_large_f16_to_f32(
+// CHECK-SAME: %[[IN:.+]]: vector<400xf16>
+func.func @cvt_large_f16_to_f32(%in : vector<400xf16>) -> vector<400xf32> {
+ // CHECK-NOT: nvvm.convert
+ // CHECK: llvm.fpext %[[IN]] : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf16> to vector<400xf32>
+ return %out : vector<400xf32>
+}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
new file mode 100644
index 0000000000000..d6c862fa5a7a6
--- /dev/null
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
@@ -0,0 +1,76 @@
+// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
+
+// Large-vector smoke tests for nvgpu.convert.fptrunc.
+
+// CHECK-LABEL: @cvt_large_f32_to_f16(
+// CHECK-SAME: %[[IN:.+]]: vector<400xf32>
+func.func @cvt_large_f32_to_f16(%in : vector<400xf32>) -> vector<400xf16> {
+ // CHECK: llvm.bitcast %[[IN]] : vector<400xf32> to vector<400xi32>
+ // CHECK: llvm.mlir.undef : vector<200xi32>
+ // CHECK-COUNT-200: nvvm.convert.f32x2.to.f16x2
+ // CHECK-NOT: nvvm.convert.f32x2.to.f16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf16>
+ return %out : vector<400xf16>
+}
+
+// CHECK-LABEL: @cvt_large_f32_to_bf16(
+func.func @cvt_large_f32_to_bf16(%in : vector<400xf32>) -> vector<400xbf16> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xf32> to vector<400xi32>
+ // CHECK-COUNT-200: nvvm.convert.f32x2.to.bf16x2
+ // CHECK-NOT: nvvm.convert.f32x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xbf16>
+ return %out : vector<400xbf16>
+}
+
+// CHECK-LABEL: @cvt_large_f32_to_f8(
+func.func @cvt_large_f32_to_f8(%in : vector<400xf32>) -> vector<400xf8E4M3FN> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xf32> to vector<400xi32>
+ // CHECK: llvm.mlir.undef : vector<100xi32>
+ // CHECK-COUNT-200: nvvm.convert.f32x2.to.f8x2
+ // CHECK-NOT: nvvm.convert.f32x2.to.f8x2
+ // CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf8E4M3FN>
+ return %out : vector<400xf8E4M3FN>
+}
+
+// CHECK-LABEL: @cvt_large_f32_to_f6(
+func.func @cvt_large_f32_to_f6(%in : vector<400xf32>) -> vector<400xf6E2M3FN> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xf32> to vector<400xi32>
+ // CHECK-COUNT-200: nvvm.convert.f32x2.to.f6x2
+ // CHECK-NOT: nvvm.convert.f32x2.to.f6x2
+ // CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
+ // CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf6E2M3FN>
+ return %out : vector<400xf6E2M3FN>
+}
+
+// CHECK-LABEL: @cvt_large_f32_to_f4(
+func.func @cvt_large_f32_to_f4(%in : vector<400xf32>) -> vector<400xf4E2M1FN> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xf32> to vector<400xi32>
+ // CHECK: llvm.mlir.undef : vector<50xi32>
+ // CHECK-COUNT-200: nvvm.convert.f32x2.to.f4x2
+ // CHECK-NOT: nvvm.convert.f32x2.to.f4x2
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf4E2M1FN>
+ return %out : vector<400xf4E2M1FN>
+}
+
+// CHECK-LABEL: @cvt_large_f16_to_f8(
+func.func @cvt_large_f16_to_f8(%in : vector<400xf16>) -> vector<400xf8E4M3FN> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xf16> to vector<200xi32>
+ // CHECK-COUNT-200: nvvm.convert.f16x2.to.f8x2
+ // CHECK-NOT: nvvm.convert.f16x2.to.f8x2
+ %out = nvgpu.convert.fptrunc %in : vector<400xf16> to vector<400xf8E4M3FN>
+ return %out : vector<400xf8E4M3FN>
+}
+
+// CHECK-LABEL: @cvt_large_bf16_to_f6(
+func.func @cvt_large_bf16_to_f6(%in : vector<400xbf16>) -> vector<400xf6E3M2FN> {
+ // CHECK: llvm.bitcast %{{.*}} : vector<400xbf16> to vector<200xi32>
+ // CHECK-COUNT-200: nvvm.convert.bf16x2.to.f6x2
+ // CHECK-NOT: nvvm.convert.bf16x2.to.f6x2
+ // CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
+ %out = nvgpu.convert.fptrunc %in : vector<400xbf16> to vector<400xf6E3M2FN>
+ return %out : vector<400xf6E3M2FN>
+}
>From 8345ebf05e15824210ea60c40901795f2c5dd69f Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Thu, 11 Jun 2026 12:49:26 +0000
Subject: [PATCH 05/11] combine Ops and address comments
---
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 73 +--
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 589 +++++++++---------
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 103 ++-
.../nvgpu-convert-fpext-large.mlir | 16 +-
.../NVGPUToNVVM/nvgpu-convert-fpext.mlir | 99 ++-
.../nvgpu-convert-fptrunc-large.mlir | 16 +-
.../NVGPUToNVVM/nvgpu-convert-fptrunc.mlir | 106 +++-
.../NVGPU/nvgpu-convert-fpext-invalid.mlir | 52 +-
.../NVGPU/nvgpu-convert-fptrunc-invalid.mlir | 77 ++-
9 files changed, 611 insertions(+), 520 deletions(-)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 14929e6187d54..0c5bd29e38336 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -677,32 +677,47 @@ def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
def AnyI32Like : TypeOrValueSemanticsContainer<I32, "scalar i32 or vector of i32">;
-def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
- [Pure]> {
- let summary = "Truncate floating-point to narrower floating-point";
+// nvgpu.convert.float only supports the satfinite and none saturation modes.
+def NVGPU_SaturationModeSatfiniteOrNone :
+ ConfinedAttr<SaturationModeAttr, [EnumAttrIsOneOf<SaturationModeAttr,
+ [SaturationModeNone, SaturationModeFinite]>]>;
+
+def NVGPU_ConvertFloatOp : NVGPU_Op<"convert.float", [Pure]> {
+ let summary = "Convert between floating-point types of different widths";
let description = [{
- Truncate a floating-point value to a smaller floating-point type.
- Destination must be strictly narrower than source.
+ Convert a floating-point value to a floating-point type of a different
+ width. The direction is inferred from the bitwidths: a narrower result is a
+ truncation, a wider result is an extension. Source and destination must have
+ different bitwidths.
- Supported paths:
+ Supported truncation paths:
+ f64 -> f32, f16, bf16
f32 -> f16, bf16, f8, f6, f4
f16 -> f8, f6, f4
bf16 -> f8, f6, f4
- The `random_bits` operand enables stochastic rounding (RS mode)
- for f32->f16/bf16 conversions. When provided, `rnd` must be RS.
+ Supported extension paths:
+ f8 -> f16, bf16
+ f6 -> f16, bf16
+ f4 -> f16, bf16
+ f16 -> f32
+ bf16 -> f32
+
+ The `sat` and `random_bits` operands apply to truncation only. The
+ `random_bits` operand enables stochastic rounding (RS mode) for
+ f32->f16/bf16 conversions; when provided, `rnd` must be RS.
Example:
```mlir
- %r = nvgpu.cvt_fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
- %r = nvgpu.cvt_fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
- %r = nvgpu.cvt_fptrunc %in : f32 to f16
- %r = nvgpu.cvt_fptrunc %in : vector<2x4xf16> to vector<2x4xf8E5M2>
+ %r = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %r = nvgpu.convert.float %in : f32 to f16
+ %r = nvgpu.convert.float %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %r = nvgpu.convert.float %in : vector<4xf8E5M2> to vector<4xf32>
```
}];
let arguments = (ins FloatLike:$in,
DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
- DefaultValuedAttr<SaturationModeAttr, "NVVM::SaturationMode::SATFINITE">:$sat,
+ DefaultValuedAttr<NVGPU_SaturationModeSatfiniteOrNone, "NVVM::SaturationMode::SATFINITE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
Optional<I32>:$random_bits
);
@@ -711,36 +726,4 @@ def NVGPU_FPTruncOp : NVGPU_Op<"convert.fptrunc",
let hasVerifier = 1;
}
-def NVGPU_FPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
- let summary = "Extend floating-point to wider floating-point";
- let description = [{
- Extend a floating-point value to a wider floating-point type.
- Destination must be strictly wider than source.
-
- Supported paths:
- f8 -> f16, bf16
- f6 -> f16
- f4 -> f16
- f16 -> f32
- bf16 -> f32
- Narrow->f32 uses two-step lowering (NVVM op to f16/bf16 intermediate,
- then LLVM FPExt). e8m0->bf16 and e8m0->f32 supported (f32 goes through
- bf16 intermediate).
-
- Example:
- ```mlir
- %r = nvgpu.cvt_fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
- %r = nvgpu.cvt_fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
- %r = nvgpu.cvt_fpext %in : vector<4xf8E5M2> to vector<4xf32>
- %r = nvgpu.cvt_fpext %in : f8E4M3FN to f32
- ```
- }];
- let arguments = (ins FloatLike:$in,
- DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
- DefaultValuedAttr<BoolAttr, "false">:$relu);
- let results = (outs FloatLike:$out);
- let assemblyFormat = "$in attr-dict `:` type($in) `to` type($out)";
- let hasVerifier = 1;
-}
-
#endif // MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 46c7783d9f43c..b4443b30c9d7a 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1713,7 +1713,7 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
};
//===----------------------------------------------------------------------===//
-// NVGPUFPTruncOp Lowering
+// NVGPUConvertFloatOp Lowering (truncation)
//===----------------------------------------------------------------------===//
enum class FPKind { F32, BF16, F16, F8, F6, F4 };
@@ -1754,7 +1754,7 @@ static int getNumSrcI32PerConv(FPKind src) {
return src == FPKind::F32 ? 2 : 1;
}
-/// Conversion op identifier for nvgpu.convert.fptrunc lowering dispatch table.
+/// Conversion op identifier for nvgpu.convert.float truncation dispatch table.
enum class FPTruncConvOp {
F32x2_TO_F16x2,
F32x2_TO_BF16x2,
@@ -1916,147 +1916,144 @@ static Value createTruncConversion(
llvm_unreachable("unhandled FPTruncConvOp");
}
-struct NVGPUFPTruncOpLowering
- : public ConvertOpToLLVMPattern<nvgpu::FPTruncOp> {
- using ConvertOpToLLVMPattern<nvgpu::FPTruncOp>::ConvertOpToLLVMPattern;
-
- LogicalResult
- matchAndRewrite(nvgpu::FPTruncOp op, OpAdaptor adaptor,
- ConversionPatternRewriter &rewriter) const override {
- if (isa<RankedTensorType>(op.getIn().getType()))
- return rewriter.notifyMatchFailure(
- op, "tensor inputs not handled; type converter should lower first");
+static LogicalResult lowerFPTrunc(nvgpu::ConvertFloatOp op,
+ nvgpu::ConvertFloatOp::Adaptor adaptor,
+ ConversionPatternRewriter &rewriter,
+ const LLVMTypeConverter *typeConverter) {
+ MLIRContext *ctx = op.getContext();
+ ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+ IntegerType i32Ty = b.getI32Type();
+ IntegerType i64Ty = b.getI64Type();
+ static constexpr int regBits = 32;
- MLIRContext *ctx = getContext();
- ImplicitLocOpBuilder b(op->getLoc(), rewriter);
- IntegerType i32Ty = b.getI32Type();
- IntegerType i64Ty = b.getI64Type();
- static constexpr int regBits = 32;
-
- auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
- auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
- if (!srcType || srcType.getRank() != 1 || !dstType ||
- dstType.getRank() != 1)
- return rewriter.notifyMatchFailure(
- op, "expected 1-D vector; canonicalize pattern handles other shapes");
-
- auto srcElemType = srcType.getElementType();
- auto dstElemType = dstType.getElementType();
- int srcBW = srcType.getElementTypeBitWidth();
- int dstBW = dstType.getElementTypeBitWidth();
- int numElems = srcType.getNumElements();
-
- NVVM::FPRoundingModeAttr rndModeAttr = op.getRndAttr();
- NVVM::SaturationModeAttr satModeAttr = op.getSatAttr();
- auto reluBoolAttr = op.getReluAttr();
- Value randomBits = adaptor.getRandomBits();
- Type actualDstFloatType = dstElemType;
-
- // STEP 1: bitcast input vector to i32 register vector.
- // f64 source: decompose to f64->f32 (LLVM fptrunc), then lower f32->dst.
- Value input = adaptor.getIn();
- if (srcBW == 64) {
- auto f32VecTy = VectorType::get(srcType.getShape(), b.getF32Type());
- input = b.create<LLVM::FPTruncOp>(f32VecTy, input);
- srcType = f32VecTy;
- srcElemType = b.getF32Type();
- srcBW = 32;
- if (dstBW == 32) {
- rewriter.replaceOp(op, input);
- return success();
- }
+ auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
+ auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
+ if (!srcType || srcType.getRank() != 1 || !dstType || dstType.getRank() != 1)
+ return rewriter.notifyMatchFailure(
+ op, "expected 1-D vector; canonicalize pattern handles other shapes");
+
+ auto srcElemType = srcType.getElementType();
+ auto dstElemType = dstType.getElementType();
+ int srcBW = srcType.getElementTypeBitWidth();
+ int dstBW = dstType.getElementTypeBitWidth();
+ int numElems = srcType.getNumElements();
+
+ NVVM::FPRoundingModeAttr rndModeAttr = op.getRndAttr();
+ NVVM::SaturationModeAttr satModeAttr = op.getSatAttr();
+ auto reluBoolAttr = op.getReluAttr();
+ Value randomBits = adaptor.getRandomBits();
+ Type actualDstFloatType = dstElemType;
+
+ // STEP 1: bitcast input vector to i32 register vector.
+ // f64 -> f32/f16/bf16 lowers to a single direct LLVM fptrunc
+ // f64 -> f8/f6/f4 first truncates to f32 and then reuses the narrow
+ // conversion path below.
+ Value input = adaptor.getIn();
+ if (srcBW == 64) {
+ if (dstBW >= 16) {
+ Type convertedType = typeConverter->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ Value result = b.create<LLVM::FPTruncOp>(convertedType, input);
+ rewriter.replaceOp(op, result);
+ return success();
}
+ auto f32VecTy = VectorType::get(srcType.getShape(), b.getF32Type());
+ input = b.create<LLVM::FPTruncOp>(f32VecTy, input);
+ srcType = f32VecTy;
+ srcElemType = b.getF32Type();
+ srcBW = 32;
+ }
- // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
- int effectiveDstBW = getEffectiveBitWidth(dstBW);
-
- int srcI32Elems = numElems * srcBW / regBits;
- int dstI32Elems = numElems * effectiveDstBW / regBits;
- Value srcI32Vec =
- b.create<LLVM::BitcastOp>(VectorType::get(srcI32Elems, i32Ty), input);
- Value dstI32Vec =
- b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
-
- // STEP 2: look up the conversion op from the (srcType, dstType) table.
- auto convEntry = lookupTruncConvOp(srcElemType, dstElemType);
- if (!convEntry)
- return rewriter.notifyMatchFailure(
- op, "unsupported type combination for truncation");
- FPTruncConvOp convOp = convEntry->convOp;
- int numSrcI32PerConv = getNumSrcI32PerConv(convEntry->src);
-
- // STEP 3: pack conversion results into destination i32 vector.
- const int srcStep = srcBW / effectiveDstBW;
- const int resultBW =
- effectiveDstBW * 2; // each conversion produces 2 (packed) elements
- const int numConvsPerI32 = regBits / resultBW;
-
- for (int srcIdx = 0, dstIdx = 0; dstIdx < dstI32Elems;
- srcIdx += srcStep, dstIdx++) {
- Value dstIdxConst =
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
- Value dstValue;
-
- if (numConvsPerI32 == 1) {
- // f16/bf16 destinations
- dstValue = createTruncConversion(
- b, ctx, convOp, srcI32Vec, srcIdx, rndModeAttr, satModeAttr,
- reluBoolAttr, dstElemType, actualDstFloatType, randomBits);
- } else {
- // f8/f6/f4 destinations: pack sub-results via vector insert + bitcast.
- auto subResultType = IntegerType::get(ctx, resultBW);
- auto subVecTy = VectorType::get(numConvsPerI32, subResultType);
- Value subVec = b.create<LLVM::UndefOp>(subVecTy);
-
- int insertIdx = numConvsPerI32 - 1;
- int curStep = srcStep;
- while (curStep > 0) {
- curStep -= numSrcI32PerConv;
- Value subResult = createTruncConversion(
- b, ctx, convOp, srcI32Vec, srcIdx + curStep, rndModeAttr,
- satModeAttr, reluBoolAttr, dstElemType, actualDstFloatType,
- /*randomBits=*/Value());
- subVec = b.create<LLVM::InsertElementOp>(
- subVec, subResult,
- b.create<LLVM::ConstantOp>(i64Ty,
- b.getI64IntegerAttr(insertIdx)));
- insertIdx--;
- }
-
- dstValue = b.create<LLVM::BitcastOp>(i32Ty, subVec);
+ // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
+ int effectiveDstBW = getEffectiveBitWidth(dstBW);
+
+ int srcI32Elems = numElems * srcBW / regBits;
+ int dstI32Elems = numElems * effectiveDstBW / regBits;
+ Value srcI32Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(srcI32Elems, i32Ty), input);
+ Value dstI32Vec =
+ b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
+
+ // STEP 2: look up the conversion op from the (srcType, dstType) table.
+ auto convEntry = lookupTruncConvOp(srcElemType, dstElemType);
+ if (!convEntry)
+ return rewriter.notifyMatchFailure(
+ op, "unsupported type combination for truncation");
+ FPTruncConvOp convOp = convEntry->convOp;
+ int numSrcI32PerConv = getNumSrcI32PerConv(convEntry->src);
+
+ // STEP 3: pack conversion results into destination i32 vector.
+ const int srcStep = srcBW / effectiveDstBW;
+ const int resultBW =
+ effectiveDstBW * 2; // each conversion produces 2 (packed) elements
+ const int numConvsPerI32 = regBits / resultBW;
+
+ for (int srcIdx = 0, dstIdx = 0; dstIdx < dstI32Elems;
+ srcIdx += srcStep, dstIdx++) {
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ Value dstValue;
+
+ if (numConvsPerI32 == 1) {
+ // f16/bf16 destinations
+ dstValue = createTruncConversion(
+ b, ctx, convOp, srcI32Vec, srcIdx, rndModeAttr, satModeAttr,
+ reluBoolAttr, dstElemType, actualDstFloatType, randomBits);
+ } else {
+ // f8/f6/f4 destinations: pack sub-results via vector insert + bitcast.
+ auto subResultType = IntegerType::get(ctx, resultBW);
+ auto subVecTy = VectorType::get(numConvsPerI32, subResultType);
+ Value subVec = b.create<LLVM::UndefOp>(subVecTy);
+
+ int insertIdx = numConvsPerI32 - 1;
+ int curStep = srcStep;
+ while (curStep > 0) {
+ curStep -= numSrcI32PerConv;
+ Value subResult = createTruncConversion(
+ b, ctx, convOp, srcI32Vec, srcIdx + curStep, rndModeAttr,
+ satModeAttr, reluBoolAttr, dstElemType, actualDstFloatType,
+ /*randomBits=*/Value());
+ subVec = b.create<LLVM::InsertElementOp>(
+ subVec, subResult,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(insertIdx)));
+ insertIdx--;
}
- dstI32Vec =
- b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ dstValue = b.create<LLVM::BitcastOp>(i32Ty, subVec);
}
- // STEP 4: produce final result.
- Type convertedType = getTypeConverter()->convertType(dstType);
- assert(convertedType && "failed to convert type");
- if (convEntry->dst == FPKind::F6) {
- IntegerType i8Ty = b.getI8Type();
- auto i8VecTy = VectorType::get(numElems, i8Ty);
- Value i8Vec = b.create<LLVM::BitcastOp>(i8VecTy, dstI32Vec);
- Value truncVec = b.create<LLVM::TruncOp>(convertedType, i8Vec);
- rewriter.replaceOp(op, truncVec);
- } else {
- auto dstVec = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
- rewriter.replaceOp(op, dstVec);
- }
- return success();
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
}
-};
+
+ // STEP 4: produce final result.
+ Type convertedType = typeConverter->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ if (convEntry->dst == FPKind::F6) {
+ IntegerType i8Ty = b.getI8Type();
+ auto i8VecTy = VectorType::get(numElems, i8Ty);
+ Value i8Vec = b.create<LLVM::BitcastOp>(i8VecTy, dstI32Vec);
+ Value truncVec = b.create<LLVM::TruncOp>(convertedType, i8Vec);
+ rewriter.replaceOp(op, truncVec);
+ } else {
+ auto dstVec = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
+ rewriter.replaceOp(op, dstVec);
+ }
+ return success();
+}
//===----------------------------------------------------------------------===//
-// NVGPUFPExtOpLowering
+// NVGPUConvertFloatOp Lowering (extension)
//===----------------------------------------------------------------------===//
-/// Extension conversion op identifier for nvgpu.convert.fpext lowering.
+/// Conversion op identifier for nvgpu.convert.float extension dispatch table.
enum class FPExtConvOp {
F8x2_TO_F16x2,
F8x2_TO_BF16x2,
F6x2_TO_F16x2,
+ F6x2_TO_BF16x2,
F4x2_TO_F16x2,
+ F4x2_TO_BF16x2,
};
struct FPExtTableEntry {
@@ -2069,7 +2066,9 @@ static constexpr FPExtTableEntry kFPExtTable[] = {
{FPKind::F8, FPKind::F16, FPExtConvOp::F8x2_TO_F16x2},
{FPKind::F8, FPKind::BF16, FPExtConvOp::F8x2_TO_BF16x2},
{FPKind::F6, FPKind::F16, FPExtConvOp::F6x2_TO_F16x2},
+ {FPKind::F6, FPKind::BF16, FPExtConvOp::F6x2_TO_BF16x2},
{FPKind::F4, FPKind::F16, FPExtConvOp::F4x2_TO_F16x2},
+ {FPKind::F4, FPKind::BF16, FPExtConvOp::F4x2_TO_BF16x2},
};
static std::optional<FPExtTableEntry> lookupExtConvOp(Type srcElemType,
@@ -2098,171 +2097,195 @@ static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
switch (convOp) {
case FPExtConvOp::F8x2_TO_F16x2: {
Value r = NVVM::ConvertF8x2ToF16x2Op::create(
- b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ b, VectorType::get(2, b.getF16Type()), src, srcTyAttr, reluAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
case FPExtConvOp::F8x2_TO_BF16x2: {
Value r = NVVM::ConvertF8x2ToBF16x2Op::create(
- b, VectorType::get(2, b.getBF16Type()), src, srcTyAttr);
+ b, VectorType::get(2, b.getBF16Type()), src, extScaleFactor, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
case FPExtConvOp::F6x2_TO_F16x2: {
Value r = NVVM::ConvertF6x2ToF16x2Op::create(
- b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ b, VectorType::get(2, b.getF16Type()), src, srcTyAttr, reluAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case FPExtConvOp::F6x2_TO_BF16x2: {
+ Value r = NVVM::ConvertF6x2ToBF16x2Op::create(
+ b, VectorType::get(2, b.getBF16Type()), src, extScaleFactor, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
case FPExtConvOp::F4x2_TO_F16x2: {
Value r = NVVM::ConvertF4x2ToF16x2Op::create(
- b, VectorType::get(2, b.getF16Type()), src, reluAttr, srcTyAttr);
+ b, VectorType::get(2, b.getF16Type()), src, srcTyAttr, reluAttr);
+ return b.create<LLVM::BitcastOp>(i32Ty, r);
+ }
+ case FPExtConvOp::F4x2_TO_BF16x2: {
+ Value r = NVVM::ConvertF4x2ToBF16x2Op::create(
+ b, VectorType::get(2, b.getBF16Type()), src, extScaleFactor, srcTyAttr);
return b.create<LLVM::BitcastOp>(i32Ty, r);
}
}
llvm_unreachable("unhandled FPExtConvOp");
}
-struct NVGPUFPExtOpLowering : public ConvertOpToLLVMPattern<nvgpu::FPExtOp> {
- using ConvertOpToLLVMPattern<nvgpu::FPExtOp>::ConvertOpToLLVMPattern;
+static LogicalResult lowerFPExt(nvgpu::ConvertFloatOp op,
+ nvgpu::ConvertFloatOp::Adaptor adaptor,
+ ConversionPatternRewriter &rewriter,
+ const LLVMTypeConverter *typeConverter) {
+ MLIRContext *ctx = op.getContext();
+ ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+ IntegerType i8Ty = b.getI8Type();
+ IntegerType i16Ty = b.getI16Type();
+ IntegerType i32Ty = b.getI32Type();
+ IntegerType i64Ty = b.getI64Type();
- LogicalResult
- matchAndRewrite(nvgpu::FPExtOp op, OpAdaptor adaptor,
- ConversionPatternRewriter &rewriter) const override {
- if (isa<RankedTensorType>(op.getIn().getType()))
- return rewriter.notifyMatchFailure(
- op, "tensor inputs not handled; type converter should lower first");
+ static constexpr int regBits = 32;
+ auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
+ auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
+ if (!srcType || srcType.getRank() != 1 || !dstType || dstType.getRank() != 1)
+ return rewriter.notifyMatchFailure(
+ op, "expected 1-D vector; canonicalize pattern handles other shapes");
+
+ auto srcElemType = srcType.getElementType();
+ auto dstElemType = dstType.getElementType();
+ int srcBW = srcType.getElementTypeBitWidth();
+ int dstBW = dstType.getElementTypeBitWidth();
+ int numElems = srcType.getNumElements();
+
+ auto reluBoolAttr = op.getReluAttr();
+ Type actualSrcFloatType = srcElemType;
+
+ assert(dstBW == 16 || dstBW == 32 || dstBW == 64);
+
+ // Wide source (f16/bf16/f32) to wide destination (f32/f64): single FPExt.
+ if (srcBW >= 16 && dstBW >= 32) {
+ Value result = adaptor.getIn();
+ if (srcElemType != dstElemType) {
+ Type convertedType = typeConverter->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ result = b.create<LLVM::FPExtOp>(convertedType, result);
+ }
+ rewriter.replaceOp(op, result);
+ return success();
+ }
- MLIRContext *ctx = getContext();
- ImplicitLocOpBuilder b(op->getLoc(), rewriter);
- IntegerType i8Ty = b.getI8Type();
- IntegerType i16Ty = b.getI16Type();
- IntegerType i32Ty = b.getI32Type();
- IntegerType i64Ty = b.getI64Type();
-
- static constexpr int regBits = 32;
- auto srcType = llvm::dyn_cast<VectorType>(op.getIn().getType());
- auto dstType = llvm::dyn_cast<VectorType>(op.getOut().getType());
- if (!srcType || srcType.getRank() != 1 || !dstType ||
- dstType.getRank() != 1)
- return rewriter.notifyMatchFailure(
- op, "expected 1-D vector; canonicalize pattern handles other shapes");
-
- auto srcElemType = srcType.getElementType();
- auto dstElemType = dstType.getElementType();
- int srcBW = srcType.getElementTypeBitWidth();
- int dstBW = dstType.getElementTypeBitWidth();
- int numElems = srcType.getNumElements();
-
- auto reluBoolAttr = op.getReluAttr();
- Type actualSrcFloatType = srcElemType;
-
- assert(dstBW == 16 || dstBW == 32 || dstBW == 64);
-
- // Wide source (f16/bf16/f32) to wide destination (f32/f64): single FPExt.
- if (srcBW >= 16 && dstBW >= 32) {
- Value result = adaptor.getIn();
- if (srcElemType != dstElemType) {
- Type convertedType = getTypeConverter()->convertType(dstType);
- assert(convertedType && "failed to convert type");
- result = b.create<LLVM::FPExtOp>(convertedType, result);
+ // Narrow source (f8/f6/f4): NVVM typed op produces f16/bf16; optionally
+ // followed by FPExt to the final f32/f64 destination.
+ bool needsFinalFPExt = (dstBW >= 32);
+ Type intermediateDstElem = dstElemType;
+ if (needsFinalFPExt && llvm::isa<Float8E8M0FNUType>(srcElemType))
+ intermediateDstElem = b.getBF16Type();
+ else if (needsFinalFPExt)
+ intermediateDstElem = b.getF16Type();
+ int intermediateDstBW = needsFinalFPExt ? 16 : dstBW;
+
+ // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
+ int effectiveSrcBW = getEffectiveBitWidth(srcBW);
+
+ // STEP 1: prepare input as i32 register vector.
+ // For f6: zext from vector<Nxi6> to vector<Nxi8>, then bitcast to i32s.
+ Value inputVec = adaptor.getIn();
+ if (srcBW == 6) {
+ auto i8VecTy = VectorType::get(numElems, i8Ty);
+ inputVec = b.create<LLVM::ZExtOp>(i8VecTy, inputVec);
+ }
+
+ int srcI32Elems = numElems * effectiveSrcBW / regBits;
+ int dstI32Elems = numElems * intermediateDstBW / regBits;
+ Value srcI32Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(srcI32Elems, i32Ty), inputVec);
+ Value dstI32Vec =
+ b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
+
+ // STEP 2: look up the conversion op from the (srcType, dstType) table.
+ auto convEntry = lookupExtConvOp(srcElemType, intermediateDstElem);
+ if (!convEntry)
+ return rewriter.notifyMatchFailure(
+ op, "unsupported type combination for extension");
+ FPExtConvOp convOp = convEntry->convOp;
+ Value extScaleFactor;
+
+ // STEP 3: iterate over source i32 elements, producing destination i32s.
+ for (int srcIdx = 0, dstIdx = 0; srcIdx < srcI32Elems; srcIdx++) {
+ Value srcI32 = b.create<LLVM::ExtractElementOp>(
+ srcI32Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(srcIdx)));
+
+ if (effectiveSrcBW == 8) {
+ // f8/f6: one i32 holds 4 bytes -> split into 2 pairs of i16 -> 2 convs.
+ Value i16Vec =
+ b.create<LLVM::BitcastOp>(VectorType::get(2, i16Ty), srcI32);
+ for (int half = 0; half < 2; half++) {
+ Value halfI16 = b.create<LLVM::ExtractElementOp>(
+ i16Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(half)));
+ Value src =
+ b.create<LLVM::BitcastOp>(VectorType::get(2, i8Ty), halfI16);
+ Value dstValue =
+ createExtConversion(b, ctx, convOp, src, reluBoolAttr,
+ actualSrcFloatType, extScaleFactor);
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ dstIdx++;
+ }
+ } else {
+ // f4: one i32 holds 4 bytes -> each byte is one conversion input.
+ Value i8Vec = b.create<LLVM::BitcastOp>(VectorType::get(4, i8Ty), srcI32);
+ for (int byteIdx = 0; byteIdx < 4; byteIdx++) {
+ Value src = b.create<LLVM::ExtractElementOp>(
+ i8Vec,
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(byteIdx)));
+ Value dstValue =
+ createExtConversion(b, ctx, convOp, src, reluBoolAttr,
+ actualSrcFloatType, extScaleFactor);
+ Value dstIdxConst =
+ b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
+ dstI32Vec =
+ b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
+ dstIdx++;
}
- rewriter.replaceOp(op, result);
- return success();
}
+ }
- // Narrow source (f8/f6/f4): NVVM typed op produces f16/bf16; optionally
- // followed by FPExt to the final f32/f64 destination.
- bool needsFinalFPExt = (dstBW >= 32);
- Type intermediateDstElem = dstElemType;
- if (needsFinalFPExt && llvm::isa<Float8E8M0FNUType>(srcElemType))
- intermediateDstElem = b.getBF16Type();
- else if (needsFinalFPExt)
- intermediateDstElem = b.getF16Type();
- int intermediateDstBW = needsFinalFPExt ? 16 : dstBW;
-
- // f6 types are 6-bit in MLIR but NVVM uses 8-bit containers.
- int effectiveSrcBW = getEffectiveBitWidth(srcBW);
-
- // STEP 1: prepare input as i32 register vector.
- // For f6: zext from vector<Nxi6> to vector<Nxi8>, then bitcast to i32s.
- Value inputVec = adaptor.getIn();
- if (srcBW == 6) {
- auto i8VecTy = VectorType::get(numElems, i8Ty);
- inputVec = b.create<LLVM::ZExtOp>(i8VecTy, inputVec);
- }
+ // STEP 4: produce final result.
+ Type convertedType = typeConverter->convertType(dstType);
+ assert(convertedType && "failed to convert type");
+ Value result;
+ if (needsFinalFPExt) {
+ auto intermediateVecTy = VectorType::get(numElems, intermediateDstElem);
+ Value intermediateVec =
+ b.create<LLVM::BitcastOp>(intermediateVecTy, dstI32Vec);
+ result = b.create<LLVM::FPExtOp>(convertedType, intermediateVec);
+ } else {
+ result = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
+ }
+ rewriter.replaceOp(op, result);
+ return success();
+}
- int srcI32Elems = numElems * effectiveSrcBW / regBits;
- int dstI32Elems = numElems * intermediateDstBW / regBits;
- Value srcI32Vec = b.create<LLVM::BitcastOp>(
- VectorType::get(srcI32Elems, i32Ty), inputVec);
- Value dstI32Vec =
- b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
+/// Lowers nvgpu.convert.float by dispatching to the truncation or extension
+/// helper based on the source/destination bitwidths.
+struct NVGPUConvertFloatOpLowering
+ : public ConvertOpToLLVMPattern<nvgpu::ConvertFloatOp> {
+ using ConvertOpToLLVMPattern<nvgpu::ConvertFloatOp>::ConvertOpToLLVMPattern;
- // STEP 2: look up the conversion op from the (srcType, dstType) table.
- auto convEntry = lookupExtConvOp(srcElemType, intermediateDstElem);
- if (!convEntry)
+ LogicalResult
+ matchAndRewrite(nvgpu::ConvertFloatOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
- op, "unsupported type combination for extension");
- FPExtConvOp convOp = convEntry->convOp;
- Value extScaleFactor;
-
- // STEP 3: iterate over source i32 elements, producing destination i32s.
- for (int srcIdx = 0, dstIdx = 0; srcIdx < srcI32Elems; srcIdx++) {
- Value srcI32 = b.create<LLVM::ExtractElementOp>(
- srcI32Vec,
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(srcIdx)));
-
- if (effectiveSrcBW == 8) {
- // f8/f6: one i32 holds 4 bytes -> split into 2 pairs of i16 -> 2 convs.
- Value i16Vec =
- b.create<LLVM::BitcastOp>(VectorType::get(2, i16Ty), srcI32);
- for (int half = 0; half < 2; half++) {
- Value halfI16 = b.create<LLVM::ExtractElementOp>(
- i16Vec,
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(half)));
- Value src =
- b.create<LLVM::BitcastOp>(VectorType::get(2, i8Ty), halfI16);
- Value dstValue =
- createExtConversion(b, ctx, convOp, src, reluBoolAttr,
- actualSrcFloatType, extScaleFactor);
- Value dstIdxConst =
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
- dstI32Vec =
- b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
- dstIdx++;
- }
- } else {
- // f4: one i32 holds 4 bytes -> each byte is one conversion input.
- Value i8Vec =
- b.create<LLVM::BitcastOp>(VectorType::get(4, i8Ty), srcI32);
- for (int byteIdx = 0; byteIdx < 4; byteIdx++) {
- Value src = b.create<LLVM::ExtractElementOp>(
- i8Vec,
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(byteIdx)));
- Value dstValue =
- createExtConversion(b, ctx, convOp, src, reluBoolAttr,
- actualSrcFloatType, extScaleFactor);
- Value dstIdxConst =
- b.create<LLVM::ConstantOp>(i64Ty, b.getI64IntegerAttr(dstIdx));
- dstI32Vec =
- b.create<LLVM::InsertElementOp>(dstI32Vec, dstValue, dstIdxConst);
- dstIdx++;
- }
- }
- }
+ op, "tensor inputs not handled; type converter should lower first");
- // STEP 4: produce final result.
- Type convertedType = getTypeConverter()->convertType(dstType);
- assert(convertedType && "failed to convert type");
- Value result;
- if (needsFinalFPExt) {
- auto intermediateVecTy = VectorType::get(numElems, intermediateDstElem);
- Value intermediateVec =
- b.create<LLVM::BitcastOp>(intermediateVecTy, dstI32Vec);
- result = b.create<LLVM::FPExtOp>(convertedType, intermediateVec);
- } else {
- result = b.create<LLVM::BitcastOp>(convertedType, dstI32Vec);
- }
- rewriter.replaceOp(op, result);
- return success();
+ int srcBW =
+ getElementTypeOrSelf(op.getIn().getType()).getIntOrFloatBitWidth();
+ int dstBW =
+ getElementTypeOrSelf(op.getOut().getType()).getIntOrFloatBitWidth();
+ if (srcBW > dstBW)
+ return lowerFPTrunc(op, adaptor, rewriter, getTypeConverter());
+ return lowerFPExt(op, adaptor, rewriter, getTypeConverter());
}
};
@@ -2278,14 +2301,14 @@ static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
return ceilDiv(padded, step) * step;
}
-/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext:
-/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
-/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
-template <typename CvtOp, bool IsTrunc>
-struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
- using OpRewritePattern<CvtOp>::OpRewritePattern;
+/// Canonicalization pattern for nvgpu.convert.float: handles scalar inputs,
+/// non-32-bit-aligned vectors, and multi-rank vectors. Runs as an
+/// OpRewritePattern on MLIR types before LLVM type conversion.
+struct NVGPUConvertFloatCanonicalizePattern
+ : public OpRewritePattern<nvgpu::ConvertFloatOp> {
+ using OpRewritePattern<nvgpu::ConvertFloatOp>::OpRewritePattern;
- LogicalResult matchAndRewrite(CvtOp op,
+ LogicalResult matchAndRewrite(nvgpu::ConvertFloatOp op,
PatternRewriter &rewriter) const override {
Type inType = op.getIn().getType();
Type outType = op.getOut().getType();
@@ -2304,7 +2327,7 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
auto srcVecTy = dyn_cast<VectorType>(inType);
bool isMultiRank = srcVecTy && srcVecTy.getRank() > 1;
int64_t numElems = isScalar ? 1 : srcVecTy.getNumElements();
- int step = IsTrunc ? effSrcBW / effDstBW : effDstBW / effSrcBW;
+ int step = srcBW > dstBW ? effSrcBW / effDstBW : effDstBW / effSrcBW;
int64_t paddedElems = computePaddedElems(numElems, srcBW, dstBW, step);
bool needsPad = (paddedElems != numElems);
@@ -2332,14 +2355,9 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
auto cvtDstTy =
VectorType::get({needsPad ? paddedElems : numElems}, dstElemTy);
- Value cvt;
- if constexpr (IsTrunc)
- cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(),
- op.getReluAttr(), op.getRandomBits());
- else
- cvt =
- CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getReluAttr());
- Value result = cvt;
+ Value result = nvgpu::ConvertFloatOp::create(
+ b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(), op.getReluAttr(),
+ op.getRandomBits());
if (needsPad)
result = vector::ExtractStridedSliceOp::create(
@@ -2357,11 +2375,6 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
return success();
}
};
-
-using NVGPUFPTruncCanonicalizePattern =
- NVGPUFPCanonicalizePattern<nvgpu::FPTruncOp, true>;
-using NVGPUFPExtCanonicalizePattern =
- NVGPUFPCanonicalizePattern<nvgpu::FPExtOp, false>;
} // namespace
void mlir::nvgpu::populateCommonGPUTypeAndAttributeConversions(
@@ -2406,12 +2419,10 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUWarpgroupMmaOpLowering, // nvgpu.warpgroup.mma
NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
- NVGPUFPTruncOpLowering, // nvgpu.convert.fptrunc
- NVGPUFPExtOpLowering, // nvgpu.convert.fpext
+ NVGPUConvertFloatOpLowering, // nvgpu.convert.float
MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
- patterns.add<NVGPUFPTruncCanonicalizePattern, NVGPUFPExtCanonicalizePattern>(
- patterns.getContext());
+ patterns.add<NVGPUConvertFloatCanonicalizePattern>(patterns.getContext());
}
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 7fb68681da4c0..8c96eb7843de4 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -701,7 +701,7 @@ LogicalResult RcpOp::verify() {
}
//===----------------------------------------------------------------------===//
-// NVGPU_CvtFPTruncOp
+// NVGPU_ConvertFloatOp
//===----------------------------------------------------------------------===//
static bool isShapedContainerType(Type t) {
@@ -737,7 +737,7 @@ static LogicalResult verifyConversionShapes(Operation *op, Type inType,
return success();
}
-LogicalResult FPTruncOp::verify() {
+LogicalResult ConvertFloatOp::verify() {
Type inType = getIn().getType();
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
@@ -750,60 +750,54 @@ LogicalResult FPTruncOp::verify() {
failed(result))
return result;
- if (srcBitWidth <= dstBitWidth)
- return emitOpError("result type ")
- << dstType << " must be narrower than operand type " << srcType;
-
- if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
- return emitOpError("input type must be 64/32/16 bitwidth, but got ")
- << srcBitWidth;
-
- if (llvm::isa<Float8E8M0FNUType>(dstType)) {
- if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
- rnd != mlir::NVVM::FPRoundingMode::RP)
- return emitOpError("expects RZ or RP rounding mode when result type is "
- "e8m0, but got ")
- << getRndAttr();
- } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
- // TODO: Currently, we only support conversions which fit into a single i32
- // register. Support f32->f8/f6/f4 conversions with RS rounding.
- if (!(srcBitWidth == 32 && (dstBitWidth == 16)))
- return emitOpError("RS (stochastic) rounding is only supported for "
- "f32->f16/bf16, got ")
- << srcType << " -> " << dstType;
- if (!getRandomBits())
- return emitOpError("random_bits operand is required with RS rounding");
- } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
- return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
- }
-
- if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
- return emitOpError("random_bits can only be used with RS rounding mode");
-
- return success();
-}
-
-//===----------------------------------------------------------------------===//
-// NVGPU_CvtFPExtOp
-//===----------------------------------------------------------------------===//
-
-LogicalResult FPExtOp::verify() {
- Type inType = getIn().getType();
- Type outType = getType();
- Type srcType = getElementTypeOrSelf(inType);
- Type dstType = getElementTypeOrSelf(outType);
- int srcBitWidth = srcType.getIntOrFloatBitWidth();
- int dstBitWidth = dstType.getIntOrFloatBitWidth();
- auto rnd = getRnd();
+ if (srcBitWidth == dstBitWidth)
+ return emitOpError("input and output types must have different bitwidths, "
+ "got ")
+ << srcType << " and " << dstType;
+
+ // Truncation: destination is narrower than source.
+ if (srcBitWidth > dstBitWidth) {
+ if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
+ return emitOpError("input type must be 64/32/16 bitwidth, but got ")
+ << srcBitWidth;
+
+ if (llvm::isa<Float8E8M0FNUType>(dstType)) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
+ rnd != mlir::NVVM::FPRoundingMode::RP)
+ return emitOpError("expects RZ or RP rounding mode when result type is "
+ "e8m0, but got ")
+ << getRndAttr();
+ } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
+ // TODO: Currently, we only support conversions which fit into a single
+ // i32 register. Support f32->f8/f6/f4 conversions with RS rounding.
+ if (!(srcBitWidth == 32 && dstBitWidth == 16))
+ return emitOpError("RS (stochastic) rounding is only supported for "
+ "f32->f16/bf16, got ")
+ << srcType << " -> " << dstType;
+ if (!getRandomBits())
+ return emitOpError("random_bits operand is required with RS rounding");
+ } else if (srcType.isF64() && dstBitWidth >= 16) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RN)
+ return emitOpError("expects RN rounding mode for f64 input, but got ")
+ << getRndAttr();
+ } else if (srcBitWidth == 32 && dstBitWidth == 16) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RN &&
+ rnd != mlir::NVVM::FPRoundingMode::RZ)
+ return emitOpError(
+ "expects RN or RZ rounding mode for f32 to f16/bf16, "
+ "but got ")
+ << getRndAttr();
+ } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
+ return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
+ }
- if (auto result = verifyConversionShapes(getOperation(), inType, outType);
- failed(result))
- return result;
+ if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
+ return emitOpError("random_bits can only be used with RS rounding mode");
- if (srcBitWidth >= dstBitWidth)
- return emitOpError("result type ")
- << dstType << " must be wider than operand type " << srcType;
+ return success();
+ }
+ // Extension: destination is wider than source.
if (dstBitWidth != 16 && dstBitWidth != 32 && dstBitWidth != 64)
return emitOpError("result type must be 16, 32, or 64 bitwidth, but got ")
<< dstBitWidth;
@@ -819,6 +813,9 @@ LogicalResult FPExtOp::verify() {
if (getRelu() && llvm::isa<BFloat16Type>(dstType))
return emitOpError("relu is not supported for bf16 destination");
+ if (getRandomBits())
+ return emitOpError("random_bits is only supported for truncation");
+
return success();
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
index 35379861f8f5d..c5a4f37cdf949 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.fpext
+// Large-vector smoke tests for nvgpu.convert.float
// CHECK-LABEL: @cvt_large_f8_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf8E4M3FN>
@@ -10,7 +10,7 @@ func.func @cvt_large_f8_to_f16(%in : vector<400xf8E4M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f8x2.to.f16x2
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf8E4M3FN> to vector<400xf16>
+ %out = nvgpu.convert.float %in : vector<400xf8E4M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_e8m0_to_bf16(%in : vector<400xf8E8M0FNU>) -> vector<400xbf1
// CHECK-COUNT-200: nvvm.convert.f8x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.fpext %in : vector<400xf8E8M0FNU> to vector<400xbf16>
+ %out = nvgpu.convert.float %in : vector<400xf8E8M0FNU> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -33,7 +33,7 @@ func.func @cvt_large_f6_to_f16(%in : vector<400xf6E2M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf16>
+ %out = nvgpu.convert.float %in : vector<400xf6E2M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -44,7 +44,7 @@ func.func @cvt_large_f4_to_f16(%in : vector<400xf4E2M1FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f4x2.to.f16x2
// CHECK-NOT: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf4E2M1FN> to vector<400xf16>
+ %out = nvgpu.convert.float %in : vector<400xf4E2M1FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -54,7 +54,7 @@ func.func @cvt_large_f8_to_f32(%in : vector<400xf8E5M2>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf8E5M2> to vector<400xf32>
+ %out = nvgpu.convert.float %in : vector<400xf8E5M2> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -64,7 +64,7 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf32>
+ %out = nvgpu.convert.float %in : vector<400xf6E2M3FN> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -73,6 +73,6 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
func.func @cvt_large_f16_to_f32(%in : vector<400xf16>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert
// CHECK: llvm.fpext %[[IN]] : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.convert.float %in : vector<400xf16> to vector<400xf32>
return %out : vector<400xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
index 73aaf63641e97..dcf090955df3c 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
@@ -18,7 +18,7 @@ func.func @cvt_float_e4m3fn_to_f16(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %out = nvgpu.convert.float %in : vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -48,7 +48,7 @@ func.func @cvt_float_e5m2_to_f16(%in : vector<8xf8E5M2>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E5M2> to vector<8xf16>
+ %out = nvgpu.convert.float %in : vector<8xf8E5M2> to vector<8xf16>
return
}
@@ -70,7 +70,7 @@ func.func @cvt_float_e8m0_to_bf16(%in : vector<8xf8E8M0FNU>) {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E8M0FNU> to vector<8xbf16>
+ %out = nvgpu.convert.float %in : vector<8xf8E8M0FNU> to vector<8xbf16>
return
}
@@ -92,7 +92,7 @@ func.func @cvt_float_e2m3_to_f16(%in : vector<8xf6E2M3FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xf16>
+ %out = nvgpu.convert.float %in : vector<8xf6E2M3FN> to vector<8xf16>
return
}
@@ -117,7 +117,7 @@ func.func @cvt_float_e3m2_to_f16(%in : vector<8xf6E3M2FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
+ %out = nvgpu.convert.float %in : vector<8xf6E3M2FN> to vector<8xf16>
return
}
@@ -150,10 +150,43 @@ func.func @cvt_float_e2m1_to_f16(%in : vector<16xf4E2M1FN>) {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xf16>
- %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf16>
+ %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xf16>
return
}
+// -----
+
+// CHECK-LABEL: @cvt_float_e2m3_to_bf16(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf6E2M3FN>
+func.func @cvt_float_e2m3_to_bf16(%in : vector<8xf6E2M3FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<8xf6E2M3FN> to vector<8xi6>
+ // CHECK: llvm.zext %[[CAST]] : vector<8xi6> to vector<8xi8>
+ // CHECK: llvm.bitcast {{.*}} : vector<8xi8> to vector<2xi32>
+ // CHECK: nvvm.convert.f6x2.to.bf16x2
+ // CHECK-SAME: : vector<2xi8>(f6E2M3FN)
+ // CHECK: nvvm.convert.f6x2.to.bf16x2
+ // CHECK: nvvm.convert.f6x2.to.bf16x2
+ // CHECK: nvvm.convert.f6x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
+ %out = nvgpu.convert.float %in : vector<8xf6E2M3FN> to vector<8xbf16>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @cvt_float_e2m1_to_bf16(
+// CHECK-SAME: %[[IN:.+]]: vector<16xf4E2M1FN>
+func.func @cvt_float_e2m1_to_bf16(%in : vector<16xf4E2M1FN>) {
+ // CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[IN]] : vector<16xf4E2M1FN> to vector<16xi4>
+ // CHECK: llvm.bitcast %[[CAST]] : vector<16xi4> to vector<2xi32>
+ // CHECK: nvvm.convert.f4x2.to.bf16x2
+ // CHECK-SAME: : i8(f4E2M1FN)
+ // CHECK: nvvm.convert.f4x2.to.bf16x2
+ // CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xbf16>
+ %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xbf16>
+ return
+}
+
// Extension to f32 (two-step: narrow -> f16/bf16 -> f32).
// -----
@@ -167,7 +200,7 @@ func.func @fpext_e4m3fn_to_f32(%in : vector<4xf8E4M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xf8E4M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -180,7 +213,7 @@ func.func @fpext_e5m2_to_f32(%in : vector<4xf8E5M2>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xf8E5M2> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -193,7 +226,7 @@ func.func @fpext_e8m0_to_f32(%in : vector<4xf8E8M0FNU>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
// CHECK: llvm.fpext {{.*}} : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E8M0FNU> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xf8E8M0FNU> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -208,7 +241,7 @@ func.func @fpext_e2m3_to_f32(%in : vector<4xf6E2M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf6E2M3FN> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xf6E2M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -220,7 +253,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
// CHECK: llvm.fpext {{.*}} : vector<8xf16> to vector<8xf32>
- %out = nvgpu.convert.fpext %in : vector<8xf4E2M1FN> to vector<8xf32>
+ %out = nvgpu.convert.float %in : vector<8xf4E2M1FN> to vector<8xf32>
return %out : vector<8xf32>
}
@@ -230,7 +263,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xf16>
func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -240,7 +273,7 @@ func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xbf16>
func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xbf16> to vector<4xf32>
+ %out = nvgpu.convert.float %in : vector<4xbf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -250,7 +283,7 @@ func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.fpext %arg0 : vector<4xf16> to vector<4xf64>
+ %out = nvgpu.convert.float %arg0 : vector<4xf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -258,7 +291,7 @@ func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xbf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.fpext %arg0 : vector<4xbf16> to vector<4xf64>
+ %out = nvgpu.convert.float %arg0 : vector<4xbf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -266,7 +299,7 @@ func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
func.func @fpext_f32_to_f64(%arg0: vector<4xf32>) -> vector<4xf64> {
// CHECK-NOT: nvvm
// CHECK: llvm.fpext %{{.*}} : vector<4xf32> to vector<4xf64>
- %out = nvgpu.convert.fpext %arg0 : vector<4xf32> to vector<4xf64>
+ %out = nvgpu.convert.float %arg0 : vector<4xf32> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -276,7 +309,7 @@ func.func @fpext_f8_to_f64(%arg0: vector<4xf8E4M3FN>) -> vector<4xf64> {
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
+ %out = nvgpu.convert.float %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -285,7 +318,7 @@ func.func @fpext_f4_to_f64(%arg0: vector<8xf4E2M1FN>) -> vector<8xf64> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.fpext %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
+ %out = nvgpu.convert.float %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
return %out : vector<8xf64>
}
@@ -295,7 +328,7 @@ func.func @fpext_e2m3_to_f64(%arg0: vector<4xf6E2M3FN>) -> vector<4xf64> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.fpext %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
+ %out = nvgpu.convert.float %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -309,7 +342,7 @@ func.func @fpext_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ %out = nvgpu.convert.float %in : f8E4M3FN to f16
return %out : f16
}
@@ -324,7 +357,7 @@ func.func @fpext_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f32 from vector<1xf32>
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ %out = nvgpu.convert.float %in : f8E4M3FN to f32
return %out : f32
}
@@ -336,7 +369,7 @@ func.func @fpext_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf8E4M3FN> to vector<8xf8E4M3FN>
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -349,7 +382,7 @@ func.func @fpext_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf32>
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -361,7 +394,7 @@ func.func @fpext_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -374,7 +407,7 @@ func.func @fpext_v3_f8_to_f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
@@ -388,7 +421,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.fpext %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
+ %out = nvgpu.convert.float %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -398,7 +431,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.fpext %in {relu = true}
+ %out = nvgpu.convert.float %in {relu = true}
: vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -414,7 +447,7 @@ func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ %out = nvgpu.convert.float %in : f8E4M3FN to f16
return %out : f16
}
@@ -423,7 +456,7 @@ func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -433,7 +466,7 @@ func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -447,7 +480,7 @@ func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ %out = nvgpu.convert.float %in : f8E4M3FN to f32
return %out : f32
}
@@ -457,7 +490,7 @@ func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -468,6 +501,6 @@ func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
index d6c862fa5a7a6..2216f0e9b5c80 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.fptrunc.
+// Large-vector smoke tests for nvgpu.convert.float.
// CHECK-LABEL: @cvt_large_f32_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf32>
@@ -10,7 +10,7 @@ func.func @cvt_large_f32_to_f16(%in : vector<400xf32>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf16>
+ %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_f32_to_bf16(%in : vector<400xf32>) -> vector<400xbf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xbf16>
+ %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -31,7 +31,7 @@ func.func @cvt_large_f32_to_f8(%in : vector<400xf32>) -> vector<400xf8E4M3FN> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f8x2
// CHECK-NOT: nvvm.convert.f32x2.to.f8x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -42,7 +42,7 @@ func.func @cvt_large_f32_to_f6(%in : vector<400xf32>) -> vector<400xf6E2M3FN> {
// CHECK-NOT: nvvm.convert.f32x2.to.f6x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf6E2M3FN>
+ %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf6E2M3FN>
return %out : vector<400xf6E2M3FN>
}
@@ -52,7 +52,7 @@ func.func @cvt_large_f32_to_f4(%in : vector<400xf32>) -> vector<400xf4E2M1FN> {
// CHECK: llvm.mlir.undef : vector<50xi32>
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f4x2
// CHECK-NOT: nvvm.convert.f32x2.to.f4x2
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf4E2M1FN>
+ %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf4E2M1FN>
return %out : vector<400xf4E2M1FN>
}
@@ -61,7 +61,7 @@ func.func @cvt_large_f16_to_f8(%in : vector<400xf16>) -> vector<400xf8E4M3FN> {
// CHECK: llvm.bitcast %{{.*}} : vector<400xf16> to vector<200xi32>
// CHECK-COUNT-200: nvvm.convert.f16x2.to.f8x2
// CHECK-NOT: nvvm.convert.f16x2.to.f8x2
- %out = nvgpu.convert.fptrunc %in : vector<400xf16> to vector<400xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<400xf16> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -71,6 +71,6 @@ func.func @cvt_large_bf16_to_f6(%in : vector<400xbf16>) -> vector<400xf6E3M2FN>
// CHECK-COUNT-200: nvvm.convert.bf16x2.to.f6x2
// CHECK-NOT: nvvm.convert.bf16x2.to.f6x2
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.fptrunc %in : vector<400xbf16> to vector<400xf6E3M2FN>
+ %out = nvgpu.convert.float %in : vector<400xbf16> to vector<400xf6E3M2FN>
return %out : vector<400xf6E3M2FN>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
index bee520a61152a..8eaccebf56e00 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
@@ -14,7 +14,7 @@ func.func @cvt_float_f32_to_f16(%in : vector<4xf32>) {
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
// CHECK-SAME: : vector<2xf16>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xf16>
return
}
@@ -27,7 +27,7 @@ func.func @cvt_float_f32_to_f16_v8(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf16>
+ %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf16>
return
}
@@ -40,7 +40,7 @@ func.func @cvt_float_f32_to_bf16(%in : vector<4xf32>) {
// CHECK-SAME: : vector<2xbf16>
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xbf16>
+ %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xbf16>
return
}
@@ -66,7 +66,7 @@ func.func @cvt_float_f32_to_e4m3(%in : vector<8xf32>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -85,7 +85,7 @@ func.func @cvt_float_f16_to_e2m3(%in : vector<8xf16>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xf16> to vector<8xf6E2M3FN>
+ %out = nvgpu.convert.float %in : vector<8xf16> to vector<8xf6E2M3FN>
return
}
@@ -99,7 +99,7 @@ func.func @cvt_float_bf16_to_e3m2(%in : vector<8xbf16>) {
// CHECK-SAME: : vector<2xbf16> -> i16(f6E3M2FN)
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xbf16> to vector<8xf6E3M2FN>
+ %out = nvgpu.convert.float %in : vector<8xbf16> to vector<8xf6E3M2FN>
return
}
@@ -124,10 +124,32 @@ func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
+ %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf6E2M3FN>
return
}
+// CHECK-LABEL: @cvt_float_f32_to_e8m0_rz(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
+func.func @cvt_float_f32_to_e8m0_rz(%in : vector<8xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
+ // CHECK-SAME: : i16(f8E8M0FNU)
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<8xf32> to vector<8xf8E8M0FNU>
+ return
+}
+
+// CHECK-LABEL: @cvt_float_f32_to_e8m0_rp(
+// CHECK-SAME: %[[IN:.+]]: vector<8xf32>
+func.func @cvt_float_f32_to_e8m0_rp(%in : vector<8xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f8x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rp>
+ // CHECK-SAME: : i16(f8E8M0FNU)
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ : vector<8xf32> to vector<8xf8E8M0FNU>
+ return
+}
+
// Scalar inputs (canonicalize: broadcast + pad + extract).
// CHECK-LABEL: @fptrunc_scalar_f32_to_f16
@@ -138,7 +160,7 @@ func.func @fptrunc_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.fptrunc %in : f32 to f16
+ %out = nvgpu.convert.float %in : f32 to f16
return %out : f16
}
@@ -148,7 +170,7 @@ func.func @fptrunc_scalar_f32_to_bf16(%in : f32) -> bf16 {
// CHECK: vector.broadcast %[[IN]]
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: vector.extract
- %out = nvgpu.convert.fptrunc %in : f32 to bf16
+ %out = nvgpu.convert.float %in : f32 to bf16
return %out : bf16
}
@@ -157,7 +179,7 @@ func.func @fptrunc_scalar_f64_to_f32(%arg0: f64) -> f32 {
// CHECK: vector.broadcast
// CHECK: llvm.fptrunc
// CHECK: vector.extract
- %out = nvgpu.convert.fptrunc %arg0 : f64 to f32
+ %out = nvgpu.convert.float %arg0 : f64 to f32
return %out : f32
}
@@ -169,7 +191,7 @@ func.func @fptrunc_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.convert.float %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -179,7 +201,7 @@ func.func @fptrunc_v4x2_f32_to_f8(%in : vector<4x2xf32>) -> vector<4x2xf8E4M3FN>
// CHECK: vector.shape_cast %[[IN]] : vector<4x2xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.shape_cast {{.*}} to vector<4x2xf8E4M3FN>
- %out = nvgpu.convert.fptrunc %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
return %out : vector<4x2xf8E4M3FN>
}
@@ -191,7 +213,7 @@ func.func @fptrunc_v1f32_to_v1f16(%in : vector<1xf32>) -> vector<1xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %in : vector<1xf32> to vector<1xf16>
+ %out = nvgpu.convert.float %in : vector<1xf32> to vector<1xf16>
return %out : vector<1xf16>
}
@@ -201,7 +223,7 @@ func.func @fptrunc_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f16x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
@@ -215,7 +237,7 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.fptrunc %in : vector<3x1xf32> to vector<3x1xf16>
+ %out = nvgpu.convert.float %in : vector<3x1xf32> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -225,27 +247,27 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
func.func @fptrunc_f64_to_f32(%arg0: vector<4xf64>) -> vector<4xf32> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
// CHECK-NOT: nvvm
- %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf32>
+ %out = nvgpu.convert.float %arg0 : vector<4xf64> to vector<4xf32>
return %out : vector<4xf32>
}
// CHECK-LABEL: @fptrunc_f64_to_f16
func.func @fptrunc_f64_to_f16(%arg0: vector<2xf64>) -> vector<2xf16> {
// CHECK: vector.insert_strided_slice
- // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
- // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf16>
+ // CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xf16>
+ %out = nvgpu.convert.float %arg0 : vector<2xf64> to vector<2xf16>
return %out : vector<2xf16>
}
// CHECK-LABEL: @fptrunc_f64_to_bf16
func.func @fptrunc_f64_to_bf16(%arg0: vector<2xf64>) -> vector<2xbf16> {
// CHECK: vector.insert_strided_slice
- // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
- // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xbf16>
+ // CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xbf16>
+ %out = nvgpu.convert.float %arg0 : vector<2xf64> to vector<2xbf16>
return %out : vector<2xbf16>
}
@@ -255,7 +277,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
// CHECK: llvm.fptrunc %{{.*}} : vector<8xf64> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
+ %out = nvgpu.convert.float %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
return %out : vector<4xf8E4M3FN>
}
@@ -265,7 +287,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<satfinite>}
+ %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<satfinite>}
: vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -274,7 +296,7 @@ func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.fptrunc %in {relu = true}
+ %out = nvgpu.convert.float %in {relu = true}
: vector<4xf32> to vector<4xf16>
return
}
@@ -285,7 +307,7 @@ func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -293,7 +315,7 @@ func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xf16>
return
}
@@ -301,7 +323,7 @@ func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: satfinite
- %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<none>}
+ %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<none>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -312,7 +334,7 @@ func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.convert.float %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -321,7 +343,25 @@ func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.convert.float %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ : vector<4xf32> to vector<4xbf16>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_f16_rz
+func.func @fptrunc_f32_to_f16_rz(%in : vector<4xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.f16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<4xf32> to vector<4xf16>
+ return
+}
+
+// CHECK-LABEL: @fptrunc_f32_to_bf16_rz
+func.func @fptrunc_f32_to_bf16_rz(%in : vector<4xf32>) {
+ // CHECK: nvvm.convert.f32x2.to.bf16x2
+ // CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf32> to vector<4xbf16>
return
}
@@ -337,7 +377,7 @@ func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
- %out = nvgpu.convert.fptrunc %in : f32 to f16
+ %out = nvgpu.convert.float %in : f32 to f16
return %out : f16
}
@@ -346,7 +386,7 @@ func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
- %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.convert.float %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -356,6 +396,6 @@ func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f16x2.to.f8x2
// CHECK-E2E: return
func.func @e2e_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
- %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.convert.float %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
index 26c3afc5da358..9e1798456379f 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
@@ -2,33 +2,25 @@
// -----
-func.func @fpext_wider(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op result type 'f4E2M1FN' must be wider than operand type 'f8E5M2'}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
- return
-}
-
-// -----
-
func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op result type must be 16, 32, or 64 bitwidth, but got 8}}
- %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
+ // expected-error @+1 {{'nvgpu.convert.float' op result type must be 16, 32, or 64 bitwidth, but got 8}}
+ %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
return
}
// -----
func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op expects bf16 or f32 output type when input type is e8m0.}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E8M0FNU> to vector<16xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op expects bf16 or f32 output type when input type is e8m0.}}
+ %out = nvgpu.convert.float %in : vector<16xf8E8M0FNU> to vector<16xf16>
return
}
// -----
func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.fpext %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<16xf8E5M2> to vector<16xf16>
return
}
@@ -36,47 +28,55 @@ func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
// -----
func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op relu is not supported for bf16 destination}}
- %out = nvgpu.convert.fpext %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op relu is not supported for bf16 destination}}
+ %out = nvgpu.convert.float %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
+ return
+}
+
+// -----
+
+func.func @fpext_random_bits(%in : vector<8xf8E5M2>, %rbits : i32) {
+ // expected-error @+1 {{'nvgpu.convert.float' op random_bits is only supported for truncation}}
+ %out = nvgpu.convert.float %in, %rbits : vector<8xf8E5M2> to vector<8xf16>
return
}
// -----
func.func @fpext_shape_mismatch(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output shapes must match}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<8xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output shapes must match}}
+ %out = nvgpu.convert.float %in : vector<16xf8E5M2> to vector<8xf16>
return
}
// -----
func.func @fpext_scalar_vector_mismatch(%in : f8E4M3FN) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.fpext %in : f8E4M3FN to vector<1xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.float %in : f8E4M3FN to vector<1xf16>
return
}
// -----
func.func @fpext_rank0_tensor(%in : tensor<f8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.fpext %in : tensor<f8E4M3FN> to tensor<f16>
+ // expected-error @+1 {{'nvgpu.convert.float' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.float %in : tensor<f8E4M3FN> to tensor<f16>
return
}
// -----
func.func @fpext_container_mismatch(%in : vector<4xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to tensor<4xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.float %in : vector<4xf8E4M3FN> to tensor<4xf16>
return
}
// -----
func.func @fpext_unranked_tensor(%in : tensor<*xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op unranked tensor types are not supported}}
- %out = nvgpu.convert.fpext %in : tensor<*xf8E4M3FN> to tensor<*xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.float %in : tensor<*xf8E4M3FN> to tensor<*xf16>
return
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
index ff899d341c99a..2ef54221e4e21 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
@@ -2,34 +2,61 @@
// -----
-func.func @fptrunc_narrower(%in : vector<16xf16>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type 'f32' must be narrower than operand type 'f16'}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf16> to vector<16xf32>
+func.func @convert_float_same_bitwidth(%in : vector<16xf16>) {
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output types must have different bitwidths, got 'f16' and 'bf16'}}
+ %out = nvgpu.convert.float %in : vector<16xf16> to vector<16xbf16>
return
}
// -----
func.func @fptrunc_src_bitwidth(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input type must be 64/32/16 bitwidth, but got 8}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ // expected-error @+1 {{'nvgpu.convert.float' op input type must be 64/32/16 bitwidth, but got 8}}
+ %out = nvgpu.convert.float %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
return
}
// -----
func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rn>}
+ // expected-error @+1 {{'nvgpu.convert.float' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rn>}
: vector<16xf32> to vector<16xf8E8M0FNU>
return
}
// -----
+func.func @fptrunc_unsupported_sat_mode(%in : vector<8xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.float' op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<sat>}
+ : vector<8xf32> to vector<8xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_f32_to_f8_rz(%in : vector<16xf32>) {
+ // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<16xf32> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fptrunc_f64_to_f16_rz(%in : vector<4xf64>) {
+ // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode for f64 input, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<4xf64> to vector<4xf16>
+ return
+}
+
+// -----
+
func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.convert.float' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -37,8 +64,8 @@ func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
// -----
func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits operand is required with RS rounding}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.convert.float' op random_bits operand is required with RS rounding}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -46,8 +73,8 @@ func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
// -----
func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
+ %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rp>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -55,8 +82,8 @@ func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
// -----
func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits can only be used with RS rounding mode}}
- %out = nvgpu.convert.fptrunc %in, %rbits
+ // expected-error @+1 {{'nvgpu.convert.float' op random_bits can only be used with RS rounding mode}}
+ %out = nvgpu.convert.float %in, %rbits
: vector<4xf32> to vector<4xf16>
return
}
@@ -64,39 +91,39 @@ func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
// -----
func.func @fptrunc_shape_mismatch(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output shapes must match}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<8xf8E4M3FN>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output shapes must match}}
+ %out = nvgpu.convert.float %in : vector<16xf32> to vector<8xf8E4M3FN>
return
}
// -----
func.func @fptrunc_scalar_vector_mismatch(%in : f32) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.fptrunc %in : f32 to vector<1xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.float %in : f32 to vector<1xf16>
return
}
// -----
func.func @fptrunc_rank0_tensor(%in : tensor<f32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.fptrunc %in : tensor<f32> to tensor<f16>
+ // expected-error @+1 {{'nvgpu.convert.float' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.float %in : tensor<f32> to tensor<f16>
return
}
// -----
func.func @fptrunc_container_mismatch(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to tensor<4xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.float %in : vector<4xf32> to tensor<4xf16>
return
}
// -----
func.func @fptrunc_unranked_tensor(%in : tensor<*xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op unranked tensor types are not supported}}
- %out = nvgpu.convert.fptrunc %in : tensor<*xf32> to tensor<*xf16>
+ // expected-error @+1 {{'nvgpu.convert.float' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.float %in : tensor<*xf32> to tensor<*xf16>
return
}
>From d68e1aa32f62c683726d2827460e805bc4772968 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 15 Jun 2026 13:57:39 +0000
Subject: [PATCH 06/11] split Op back to convert.fptrunc and convert.fpext
---
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 61 ++++++----
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 83 ++++++++-----
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 113 ++++++++++--------
.../nvgpu-convert-fpext-large.mlir | 16 +--
.../NVGPUToNVVM/nvgpu-convert-fpext.mlir | 70 +++++------
.../nvgpu-convert-fptrunc-large.mlir | 16 +--
.../NVGPUToNVVM/nvgpu-convert-fptrunc.mlir | 66 +++++-----
.../NVGPU/nvgpu-convert-fpext-invalid.mlir | 52 ++++----
.../NVGPU/nvgpu-convert-fptrunc-invalid.mlir | 62 +++++-----
9 files changed, 294 insertions(+), 245 deletions(-)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 0c5bd29e38336..9f27db4c9dcc2 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -677,42 +677,32 @@ def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
def AnyI32Like : TypeOrValueSemanticsContainer<I32, "scalar i32 or vector of i32">;
-// nvgpu.convert.float only supports the satfinite and none saturation modes.
+// nvgpu.convert.fptrunc only supports the satfinite and none saturation modes.
def NVGPU_SaturationModeSatfiniteOrNone :
ConfinedAttr<SaturationModeAttr, [EnumAttrIsOneOf<SaturationModeAttr,
[SaturationModeNone, SaturationModeFinite]>]>;
-def NVGPU_ConvertFloatOp : NVGPU_Op<"convert.float", [Pure]> {
- let summary = "Convert between floating-point types of different widths";
+def NVGPU_ConvertFPTruncOp : NVGPU_Op<"convert.fptrunc", [Pure]> {
+ let summary = "Truncate floating-point to narrower floating-point";
let description = [{
- Convert a floating-point value to a floating-point type of a different
- width. The direction is inferred from the bitwidths: a narrower result is a
- truncation, a wider result is an extension. Source and destination must have
- different bitwidths.
+ Truncate a floating-point value to a smaller floating-point type.
+ Destination must be strictly narrower than source.
- Supported truncation paths:
+ Supported paths:
f64 -> f32, f16, bf16
f32 -> f16, bf16, f8, f6, f4
f16 -> f8, f6, f4
bf16 -> f8, f6, f4
- Supported extension paths:
- f8 -> f16, bf16
- f6 -> f16, bf16
- f4 -> f16, bf16
- f16 -> f32
- bf16 -> f32
-
- The `sat` and `random_bits` operands apply to truncation only. The
- `random_bits` operand enables stochastic rounding (RS mode) for
+ The `random_bits` operand enables stochastic rounding (RS mode) for
f32->f16/bf16 conversions; when provided, `rnd` must be RS.
Example:
```mlir
- %r = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
- %r = nvgpu.convert.float %in : f32 to f16
- %r = nvgpu.convert.float %in : vector<8xf8E4M3FN> to vector<8xf16>
- %r = nvgpu.convert.float %in : vector<4xf8E5M2> to vector<4xf32>
+ %r = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %r = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
+ %r = nvgpu.convert.fptrunc %in : f32 to f16
+ %r = nvgpu.convert.fptrunc %in : vector<2x4xf16> to vector<2x4xf8E5M2>
```
}];
let arguments = (ins FloatLike:$in,
@@ -726,4 +716,33 @@ def NVGPU_ConvertFloatOp : NVGPU_Op<"convert.float", [Pure]> {
let hasVerifier = 1;
}
+def NVGPU_ConvertFPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
+ let summary = "Extend floating-point to wider floating-point";
+ let description = [{
+ Extend a floating-point value to a wider floating-point type.
+ Destination must be strictly wider than source.
+
+ Supported paths:
+ f8 -> f16, bf16
+ f6 -> f16, bf16
+ f4 -> f16, bf16
+ f16 -> f32
+ bf16 -> f32
+
+ Example:
+ ```mlir
+ %r = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %r = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
+ %r = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
+ %r = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ ```
+ }];
+ let arguments = (ins FloatLike:$in,
+ DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
+ DefaultValuedAttr<BoolAttr, "false">:$relu);
+ let results = (outs FloatLike:$out);
+ let assemblyFormat = "$in attr-dict `:` type($in) `to` type($out)";
+ let hasVerifier = 1;
+}
+
#endif // MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index b4443b30c9d7a..abd6ac7ff8fe4 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1713,7 +1713,7 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
};
//===----------------------------------------------------------------------===//
-// NVGPUConvertFloatOp Lowering (truncation)
+// NVGPUConvertFPTruncOp Lowering
//===----------------------------------------------------------------------===//
enum class FPKind { F32, BF16, F16, F8, F6, F4 };
@@ -1754,7 +1754,7 @@ static int getNumSrcI32PerConv(FPKind src) {
return src == FPKind::F32 ? 2 : 1;
}
-/// Conversion op identifier for nvgpu.convert.float truncation dispatch table.
+/// Conversion op identifier for nvgpu.convert.fptrunc lowering dispatch table.
enum class FPTruncConvOp {
F32x2_TO_F16x2,
F32x2_TO_BF16x2,
@@ -1916,8 +1916,8 @@ static Value createTruncConversion(
llvm_unreachable("unhandled FPTruncConvOp");
}
-static LogicalResult lowerFPTrunc(nvgpu::ConvertFloatOp op,
- nvgpu::ConvertFloatOp::Adaptor adaptor,
+static LogicalResult lowerFPTrunc(nvgpu::ConvertFPTruncOp op,
+ nvgpu::ConvertFPTruncOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter,
const LLVMTypeConverter *typeConverter) {
MLIRContext *ctx = op.getContext();
@@ -2043,10 +2043,10 @@ static LogicalResult lowerFPTrunc(nvgpu::ConvertFloatOp op,
}
//===----------------------------------------------------------------------===//
-// NVGPUConvertFloatOp Lowering (extension)
+// NVGPUConvertFPExtOp Lowering
//===----------------------------------------------------------------------===//
-/// Conversion op identifier for nvgpu.convert.float extension dispatch table.
+/// Conversion op identifier for nvgpu.convert.fpext lowering dispatch table.
enum class FPExtConvOp {
F8x2_TO_F16x2,
F8x2_TO_BF16x2,
@@ -2129,8 +2129,8 @@ static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
llvm_unreachable("unhandled FPExtConvOp");
}
-static LogicalResult lowerFPExt(nvgpu::ConvertFloatOp op,
- nvgpu::ConvertFloatOp::Adaptor adaptor,
+static LogicalResult lowerFPExt(nvgpu::ConvertFPExtOp op,
+ nvgpu::ConvertFPExtOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter,
const LLVMTypeConverter *typeConverter) {
MLIRContext *ctx = op.getContext();
@@ -2266,25 +2266,30 @@ static LogicalResult lowerFPExt(nvgpu::ConvertFloatOp op,
return success();
}
-/// Lowers nvgpu.convert.float by dispatching to the truncation or extension
-/// helper based on the source/destination bitwidths.
-struct NVGPUConvertFloatOpLowering
- : public ConvertOpToLLVMPattern<nvgpu::ConvertFloatOp> {
- using ConvertOpToLLVMPattern<nvgpu::ConvertFloatOp>::ConvertOpToLLVMPattern;
+struct NVGPUConvertFPTruncOpLowering
+ : public ConvertOpToLLVMPattern<nvgpu::ConvertFPTruncOp> {
+ using ConvertOpToLLVMPattern<nvgpu::ConvertFPTruncOp>::ConvertOpToLLVMPattern;
LogicalResult
- matchAndRewrite(nvgpu::ConvertFloatOp op, OpAdaptor adaptor,
+ matchAndRewrite(nvgpu::ConvertFPTruncOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
op, "tensor inputs not handled; type converter should lower first");
+ return lowerFPTrunc(op, adaptor, rewriter, getTypeConverter());
+ }
+};
+
+struct NVGPUConvertFPExtOpLowering
+ : public ConvertOpToLLVMPattern<nvgpu::ConvertFPExtOp> {
+ using ConvertOpToLLVMPattern<nvgpu::ConvertFPExtOp>::ConvertOpToLLVMPattern;
- int srcBW =
- getElementTypeOrSelf(op.getIn().getType()).getIntOrFloatBitWidth();
- int dstBW =
- getElementTypeOrSelf(op.getOut().getType()).getIntOrFloatBitWidth();
- if (srcBW > dstBW)
- return lowerFPTrunc(op, adaptor, rewriter, getTypeConverter());
+ LogicalResult
+ matchAndRewrite(nvgpu::ConvertFPExtOp op, OpAdaptor adaptor,
+ ConversionPatternRewriter &rewriter) const override {
+ if (isa<RankedTensorType>(op.getIn().getType()))
+ return rewriter.notifyMatchFailure(
+ op, "tensor inputs not handled; type converter should lower first");
return lowerFPExt(op, adaptor, rewriter, getTypeConverter());
}
};
@@ -2301,14 +2306,14 @@ static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
return ceilDiv(padded, step) * step;
}
-/// Canonicalization pattern for nvgpu.convert.float: handles scalar inputs,
-/// non-32-bit-aligned vectors, and multi-rank vectors. Runs as an
-/// OpRewritePattern on MLIR types before LLVM type conversion.
-struct NVGPUConvertFloatCanonicalizePattern
- : public OpRewritePattern<nvgpu::ConvertFloatOp> {
- using OpRewritePattern<nvgpu::ConvertFloatOp>::OpRewritePattern;
+/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext:
+/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
+/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
+template <typename CvtOp, bool IsTrunc>
+struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
+ using OpRewritePattern<CvtOp>::OpRewritePattern;
- LogicalResult matchAndRewrite(nvgpu::ConvertFloatOp op,
+ LogicalResult matchAndRewrite(CvtOp op,
PatternRewriter &rewriter) const override {
Type inType = op.getIn().getType();
Type outType = op.getOut().getType();
@@ -2327,7 +2332,7 @@ struct NVGPUConvertFloatCanonicalizePattern
auto srcVecTy = dyn_cast<VectorType>(inType);
bool isMultiRank = srcVecTy && srcVecTy.getRank() > 1;
int64_t numElems = isScalar ? 1 : srcVecTy.getNumElements();
- int step = srcBW > dstBW ? effSrcBW / effDstBW : effDstBW / effSrcBW;
+ int step = IsTrunc ? effSrcBW / effDstBW : effDstBW / effSrcBW;
int64_t paddedElems = computePaddedElems(numElems, srcBW, dstBW, step);
bool needsPad = (paddedElems != numElems);
@@ -2355,9 +2360,14 @@ struct NVGPUConvertFloatCanonicalizePattern
auto cvtDstTy =
VectorType::get({needsPad ? paddedElems : numElems}, dstElemTy);
- Value result = nvgpu::ConvertFloatOp::create(
- b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(), op.getReluAttr(),
- op.getRandomBits());
+ Value cvt;
+ if constexpr (IsTrunc)
+ cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(),
+ op.getReluAttr(), op.getRandomBits());
+ else
+ cvt =
+ CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getReluAttr());
+ Value result = cvt;
if (needsPad)
result = vector::ExtractStridedSliceOp::create(
@@ -2375,6 +2385,11 @@ struct NVGPUConvertFloatCanonicalizePattern
return success();
}
};
+
+using NVGPUConvertFPTruncCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::ConvertFPTruncOp, true>;
+using NVGPUConvertFPExtCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::ConvertFPExtOp, false>;
} // namespace
void mlir::nvgpu::populateCommonGPUTypeAndAttributeConversions(
@@ -2419,10 +2434,12 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUWarpgroupMmaOpLowering, // nvgpu.warpgroup.mma
NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
- NVGPUConvertFloatOpLowering, // nvgpu.convert.float
+ NVGPUConvertFPTruncOpLowering, // nvgpu.convert.fptrunc
+ NVGPUConvertFPExtOpLowering, // nvgpu.convert.fpext
MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
- patterns.add<NVGPUConvertFloatCanonicalizePattern>(patterns.getContext());
+ patterns.add<NVGPUConvertFPTruncCanonicalizePattern,
+ NVGPUConvertFPExtCanonicalizePattern>(patterns.getContext());
}
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 8c96eb7843de4..98973a733f063 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -701,7 +701,7 @@ LogicalResult RcpOp::verify() {
}
//===----------------------------------------------------------------------===//
-// NVGPU_ConvertFloatOp
+// NVGPU_ConvertFPTruncOp
//===----------------------------------------------------------------------===//
static bool isShapedContainerType(Type t) {
@@ -737,7 +737,7 @@ static LogicalResult verifyConversionShapes(Operation *op, Type inType,
return success();
}
-LogicalResult ConvertFloatOp::verify() {
+LogicalResult ConvertFPTruncOp::verify() {
Type inType = getIn().getType();
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
@@ -750,54 +750,70 @@ LogicalResult ConvertFloatOp::verify() {
failed(result))
return result;
- if (srcBitWidth == dstBitWidth)
- return emitOpError("input and output types must have different bitwidths, "
- "got ")
- << srcType << " and " << dstType;
-
- // Truncation: destination is narrower than source.
- if (srcBitWidth > dstBitWidth) {
- if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
- return emitOpError("input type must be 64/32/16 bitwidth, but got ")
- << srcBitWidth;
-
- if (llvm::isa<Float8E8M0FNUType>(dstType)) {
- if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
- rnd != mlir::NVVM::FPRoundingMode::RP)
- return emitOpError("expects RZ or RP rounding mode when result type is "
- "e8m0, but got ")
- << getRndAttr();
- } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
- // TODO: Currently, we only support conversions which fit into a single
- // i32 register. Support f32->f8/f6/f4 conversions with RS rounding.
- if (!(srcBitWidth == 32 && dstBitWidth == 16))
- return emitOpError("RS (stochastic) rounding is only supported for "
- "f32->f16/bf16, got ")
- << srcType << " -> " << dstType;
- if (!getRandomBits())
- return emitOpError("random_bits operand is required with RS rounding");
- } else if (srcType.isF64() && dstBitWidth >= 16) {
- if (rnd != mlir::NVVM::FPRoundingMode::RN)
- return emitOpError("expects RN rounding mode for f64 input, but got ")
- << getRndAttr();
- } else if (srcBitWidth == 32 && dstBitWidth == 16) {
- if (rnd != mlir::NVVM::FPRoundingMode::RN &&
- rnd != mlir::NVVM::FPRoundingMode::RZ)
- return emitOpError(
- "expects RN or RZ rounding mode for f32 to f16/bf16, "
- "but got ")
- << getRndAttr();
- } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
- return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
- }
+ if (srcBitWidth <= dstBitWidth)
+ return emitOpError("result type ")
+ << dstType << " must be narrower than operand type " << srcType;
+
+ if (!(srcBitWidth == 64 || srcBitWidth == 32 || srcBitWidth == 16))
+ return emitOpError("input type must be 64/32/16 bitwidth, but got ")
+ << srcBitWidth;
+
+ if (llvm::isa<Float8E8M0FNUType>(dstType)) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RZ &&
+ rnd != mlir::NVVM::FPRoundingMode::RP)
+ return emitOpError("expects RZ or RP rounding mode when result type is "
+ "e8m0, but got ")
+ << getRndAttr();
+ } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
+ // TODO: Currently, we only support conversions which fit into a single i32
+ // register. Support f32->f8/f6/f4 conversions with RS rounding.
+ if (!(srcBitWidth == 32 && dstBitWidth == 16))
+ return emitOpError("RS (stochastic) rounding is only supported for "
+ "f32->f16/bf16, got ")
+ << srcType << " -> " << dstType;
+ if (!getRandomBits())
+ return emitOpError("random_bits operand is required with RS rounding");
+ } else if (srcType.isF64() && dstBitWidth >= 16) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RN)
+ return emitOpError("expects RN rounding mode for f64 input, but got ")
+ << getRndAttr();
+ } else if (srcBitWidth == 32 && dstBitWidth == 16) {
+ if (rnd != mlir::NVVM::FPRoundingMode::RN &&
+ rnd != mlir::NVVM::FPRoundingMode::RZ)
+ return emitOpError("expects RN or RZ rounding mode for f32 to f16/bf16, "
+ "but got ")
+ << getRndAttr();
+ } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
+ return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
+ }
- if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
- return emitOpError("random_bits can only be used with RS rounding mode");
+ if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
+ return emitOpError("random_bits can only be used with RS rounding mode");
- return success();
- }
+ return success();
+}
+
+//===----------------------------------------------------------------------===//
+// NVGPU_ConvertFPExtOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult ConvertFPExtOp::verify() {
+ Type inType = getIn().getType();
+ Type outType = getType();
+ Type srcType = getElementTypeOrSelf(inType);
+ Type dstType = getElementTypeOrSelf(outType);
+ int srcBitWidth = srcType.getIntOrFloatBitWidth();
+ int dstBitWidth = dstType.getIntOrFloatBitWidth();
+ auto rnd = getRnd();
+
+ if (auto result = verifyConversionShapes(getOperation(), inType, outType);
+ failed(result))
+ return result;
+
+ if (srcBitWidth >= dstBitWidth)
+ return emitOpError("result type ")
+ << dstType << " must be wider than operand type " << srcType;
- // Extension: destination is wider than source.
if (dstBitWidth != 16 && dstBitWidth != 32 && dstBitWidth != 64)
return emitOpError("result type must be 16, 32, or 64 bitwidth, but got ")
<< dstBitWidth;
@@ -813,9 +829,6 @@ LogicalResult ConvertFloatOp::verify() {
if (getRelu() && llvm::isa<BFloat16Type>(dstType))
return emitOpError("relu is not supported for bf16 destination");
- if (getRandomBits())
- return emitOpError("random_bits is only supported for truncation");
-
return success();
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
index c5a4f37cdf949..35379861f8f5d 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.float
+// Large-vector smoke tests for nvgpu.convert.fpext
// CHECK-LABEL: @cvt_large_f8_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf8E4M3FN>
@@ -10,7 +10,7 @@ func.func @cvt_large_f8_to_f16(%in : vector<400xf8E4M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f8x2.to.f16x2
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.float %in : vector<400xf8E4M3FN> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E4M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_e8m0_to_bf16(%in : vector<400xf8E8M0FNU>) -> vector<400xbf1
// CHECK-COUNT-200: nvvm.convert.f8x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.float %in : vector<400xf8E8M0FNU> to vector<400xbf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E8M0FNU> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -33,7 +33,7 @@ func.func @cvt_large_f6_to_f16(%in : vector<400xf6E2M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.float %in : vector<400xf6E2M3FN> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -44,7 +44,7 @@ func.func @cvt_large_f4_to_f16(%in : vector<400xf4E2M1FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f4x2.to.f16x2
// CHECK-NOT: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.float %in : vector<400xf4E2M1FN> to vector<400xf16>
+ %out = nvgpu.convert.fpext %in : vector<400xf4E2M1FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -54,7 +54,7 @@ func.func @cvt_large_f8_to_f32(%in : vector<400xf8E5M2>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.float %in : vector<400xf8E5M2> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf8E5M2> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -64,7 +64,7 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.float %in : vector<400xf6E2M3FN> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -73,6 +73,6 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
func.func @cvt_large_f16_to_f32(%in : vector<400xf16>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert
// CHECK: llvm.fpext %[[IN]] : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.float %in : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.convert.fpext %in : vector<400xf16> to vector<400xf32>
return %out : vector<400xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
index dcf090955df3c..1d711540f7aff 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
@@ -18,7 +18,7 @@ func.func @cvt_float_e4m3fn_to_f16(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.float %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -48,7 +48,7 @@ func.func @cvt_float_e5m2_to_f16(%in : vector<8xf8E5M2>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.float %in : vector<8xf8E5M2> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E5M2> to vector<8xf16>
return
}
@@ -70,7 +70,7 @@ func.func @cvt_float_e8m0_to_bf16(%in : vector<8xf8E8M0FNU>) {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
- %out = nvgpu.convert.float %in : vector<8xf8E8M0FNU> to vector<8xbf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf8E8M0FNU> to vector<8xbf16>
return
}
@@ -92,7 +92,7 @@ func.func @cvt_float_e2m3_to_f16(%in : vector<8xf6E2M3FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.float %in : vector<8xf6E2M3FN> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xf16>
return
}
@@ -117,7 +117,7 @@ func.func @cvt_float_e3m2_to_f16(%in : vector<8xf6E3M2FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.float %in : vector<8xf6E3M2FN> to vector<8xf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
return
}
@@ -150,7 +150,7 @@ func.func @cvt_float_e2m1_to_f16(%in : vector<16xf4E2M1FN>) {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xf16>
- %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xf16>
+ %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf16>
return
}
@@ -168,7 +168,7 @@ func.func @cvt_float_e2m3_to_bf16(%in : vector<8xf6E2M3FN>) {
// CHECK: nvvm.convert.f6x2.to.bf16x2
// CHECK: nvvm.convert.f6x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
- %out = nvgpu.convert.float %in : vector<8xf6E2M3FN> to vector<8xbf16>
+ %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xbf16>
return
}
@@ -183,7 +183,7 @@ func.func @cvt_float_e2m1_to_bf16(%in : vector<16xf4E2M1FN>) {
// CHECK-SAME: : i8(f4E2M1FN)
// CHECK: nvvm.convert.f4x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xbf16>
- %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xbf16>
+ %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xbf16>
return
}
@@ -200,7 +200,7 @@ func.func @fpext_e4m3fn_to_f32(%in : vector<4xf8E4M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xf8E4M3FN> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -213,7 +213,7 @@ func.func @fpext_e5m2_to_f32(%in : vector<4xf8E5M2>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xf8E5M2> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -226,7 +226,7 @@ func.func @fpext_e8m0_to_f32(%in : vector<4xf8E8M0FNU>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
// CHECK: llvm.fpext {{.*}} : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xf8E8M0FNU> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf8E8M0FNU> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -241,7 +241,7 @@ func.func @fpext_e2m3_to_f32(%in : vector<4xf6E2M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xf6E2M3FN> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf6E2M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -253,7 +253,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
// CHECK: llvm.fpext {{.*}} : vector<8xf16> to vector<8xf32>
- %out = nvgpu.convert.float %in : vector<8xf4E2M1FN> to vector<8xf32>
+ %out = nvgpu.convert.fpext %in : vector<8xf4E2M1FN> to vector<8xf32>
return %out : vector<8xf32>
}
@@ -263,7 +263,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xf16>
func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -273,7 +273,7 @@ func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xbf16>
func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.float %in : vector<4xbf16> to vector<4xf32>
+ %out = nvgpu.convert.fpext %in : vector<4xbf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -283,7 +283,7 @@ func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.float %arg0 : vector<4xf16> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -291,7 +291,7 @@ func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xbf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.float %arg0 : vector<4xbf16> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xbf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -299,7 +299,7 @@ func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
func.func @fpext_f32_to_f64(%arg0: vector<4xf32>) -> vector<4xf64> {
// CHECK-NOT: nvvm
// CHECK: llvm.fpext %{{.*}} : vector<4xf32> to vector<4xf64>
- %out = nvgpu.convert.float %arg0 : vector<4xf32> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf32> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -309,7 +309,7 @@ func.func @fpext_f8_to_f64(%arg0: vector<4xf8E4M3FN>) -> vector<4xf64> {
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -318,7 +318,7 @@ func.func @fpext_f4_to_f64(%arg0: vector<8xf4E2M1FN>) -> vector<8xf64> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.float %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
return %out : vector<8xf64>
}
@@ -328,7 +328,7 @@ func.func @fpext_e2m3_to_f64(%arg0: vector<4xf6E2M3FN>) -> vector<4xf64> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.float %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
+ %out = nvgpu.convert.fpext %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -342,7 +342,7 @@ func.func @fpext_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.float %in : f8E4M3FN to f16
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
return %out : f16
}
@@ -357,7 +357,7 @@ func.func @fpext_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f32 from vector<1xf32>
- %out = nvgpu.convert.float %in : f8E4M3FN to f32
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
return %out : f32
}
@@ -369,7 +369,7 @@ func.func @fpext_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf8E4M3FN> to vector<8xf8E4M3FN>
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -382,7 +382,7 @@ func.func @fpext_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf32>
- %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -394,7 +394,7 @@ func.func @fpext_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -407,7 +407,7 @@ func.func @fpext_v3_f8_to_f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
@@ -421,7 +421,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.float %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
+ %out = nvgpu.convert.fpext %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -431,7 +431,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.float %in {relu = true}
+ %out = nvgpu.convert.fpext %in {relu = true}
: vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -447,7 +447,7 @@ func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
- %out = nvgpu.convert.float %in : f8E4M3FN to f16
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
return %out : f16
}
@@ -456,7 +456,7 @@ func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
- %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -466,7 +466,7 @@ func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
- %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -480,7 +480,7 @@ func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
- %out = nvgpu.convert.float %in : f8E4M3FN to f32
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
return %out : f32
}
@@ -490,7 +490,7 @@ func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
- %out = nvgpu.convert.float %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -501,6 +501,6 @@ func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
- %out = nvgpu.convert.float %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
index 2216f0e9b5c80..d6c862fa5a7a6 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.float.
+// Large-vector smoke tests for nvgpu.convert.fptrunc.
// CHECK-LABEL: @cvt_large_f32_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf32>
@@ -10,7 +10,7 @@ func.func @cvt_large_f32_to_f16(%in : vector<400xf32>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_f32_to_bf16(%in : vector<400xf32>) -> vector<400xbf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xbf16>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -31,7 +31,7 @@ func.func @cvt_large_f32_to_f8(%in : vector<400xf32>) -> vector<400xf8E4M3FN> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f8x2
// CHECK-NOT: nvvm.convert.f32x2.to.f8x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
- %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -42,7 +42,7 @@ func.func @cvt_large_f32_to_f6(%in : vector<400xf32>) -> vector<400xf6E2M3FN> {
// CHECK-NOT: nvvm.convert.f32x2.to.f6x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf6E2M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf6E2M3FN>
return %out : vector<400xf6E2M3FN>
}
@@ -52,7 +52,7 @@ func.func @cvt_large_f32_to_f4(%in : vector<400xf32>) -> vector<400xf4E2M1FN> {
// CHECK: llvm.mlir.undef : vector<50xi32>
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f4x2
// CHECK-NOT: nvvm.convert.f32x2.to.f4x2
- %out = nvgpu.convert.float %in : vector<400xf32> to vector<400xf4E2M1FN>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf4E2M1FN>
return %out : vector<400xf4E2M1FN>
}
@@ -61,7 +61,7 @@ func.func @cvt_large_f16_to_f8(%in : vector<400xf16>) -> vector<400xf8E4M3FN> {
// CHECK: llvm.bitcast %{{.*}} : vector<400xf16> to vector<200xi32>
// CHECK-COUNT-200: nvvm.convert.f16x2.to.f8x2
// CHECK-NOT: nvvm.convert.f16x2.to.f8x2
- %out = nvgpu.convert.float %in : vector<400xf16> to vector<400xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<400xf16> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -71,6 +71,6 @@ func.func @cvt_large_bf16_to_f6(%in : vector<400xbf16>) -> vector<400xf6E3M2FN>
// CHECK-COUNT-200: nvvm.convert.bf16x2.to.f6x2
// CHECK-NOT: nvvm.convert.bf16x2.to.f6x2
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.float %in : vector<400xbf16> to vector<400xf6E3M2FN>
+ %out = nvgpu.convert.fptrunc %in : vector<400xbf16> to vector<400xf6E3M2FN>
return %out : vector<400xf6E3M2FN>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
index 8eaccebf56e00..8d1c71e22f366 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
@@ -14,7 +14,7 @@ func.func @cvt_float_f32_to_f16(%in : vector<4xf32>) {
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
// CHECK-SAME: : vector<2xf16>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
- %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
return
}
@@ -27,7 +27,7 @@ func.func @cvt_float_f32_to_f16_v8(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf16>
return
}
@@ -40,7 +40,7 @@ func.func @cvt_float_f32_to_bf16(%in : vector<4xf32>) {
// CHECK-SAME: : vector<2xbf16>
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
- %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xbf16>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xbf16>
return
}
@@ -66,7 +66,7 @@ func.func @cvt_float_f32_to_e4m3(%in : vector<8xf32>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
- %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -85,7 +85,7 @@ func.func @cvt_float_f16_to_e2m3(%in : vector<8xf16>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.float %in : vector<8xf16> to vector<8xf6E2M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf16> to vector<8xf6E2M3FN>
return
}
@@ -99,7 +99,7 @@ func.func @cvt_float_bf16_to_e3m2(%in : vector<8xbf16>) {
// CHECK-SAME: : vector<2xbf16> -> i16(f6E3M2FN)
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.float %in : vector<8xbf16> to vector<8xf6E3M2FN>
+ %out = nvgpu.convert.fptrunc %in : vector<8xbf16> to vector<8xf6E3M2FN>
return
}
@@ -124,7 +124,7 @@ func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf6E2M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
return
}
@@ -134,7 +134,7 @@ func.func @cvt_float_f32_to_e8m0_rz(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
// CHECK-SAME: : i16(f8E8M0FNU)
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<8xf32> to vector<8xf8E8M0FNU>
return
}
@@ -145,7 +145,7 @@ func.func @cvt_float_f32_to_e8m0_rp(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rp>
// CHECK-SAME: : i16(f8E8M0FNU)
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
: vector<8xf32> to vector<8xf8E8M0FNU>
return
}
@@ -160,7 +160,7 @@ func.func @fptrunc_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.float %in : f32 to f16
+ %out = nvgpu.convert.fptrunc %in : f32 to f16
return %out : f16
}
@@ -170,7 +170,7 @@ func.func @fptrunc_scalar_f32_to_bf16(%in : f32) -> bf16 {
// CHECK: vector.broadcast %[[IN]]
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: vector.extract
- %out = nvgpu.convert.float %in : f32 to bf16
+ %out = nvgpu.convert.fptrunc %in : f32 to bf16
return %out : bf16
}
@@ -179,7 +179,7 @@ func.func @fptrunc_scalar_f64_to_f32(%arg0: f64) -> f32 {
// CHECK: vector.broadcast
// CHECK: llvm.fptrunc
// CHECK: vector.extract
- %out = nvgpu.convert.float %arg0 : f64 to f32
+ %out = nvgpu.convert.fptrunc %arg0 : f64 to f32
return %out : f32
}
@@ -191,7 +191,7 @@ func.func @fptrunc_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.float %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -201,7 +201,7 @@ func.func @fptrunc_v4x2_f32_to_f8(%in : vector<4x2xf32>) -> vector<4x2xf8E4M3FN>
// CHECK: vector.shape_cast %[[IN]] : vector<4x2xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.shape_cast {{.*}} to vector<4x2xf8E4M3FN>
- %out = nvgpu.convert.float %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
return %out : vector<4x2xf8E4M3FN>
}
@@ -213,7 +213,7 @@ func.func @fptrunc_v1f32_to_v1f16(%in : vector<1xf32>) -> vector<1xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %in : vector<1xf32> to vector<1xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<1xf32> to vector<1xf16>
return %out : vector<1xf16>
}
@@ -223,7 +223,7 @@ func.func @fptrunc_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f16x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
@@ -237,7 +237,7 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.float %in : vector<3x1xf32> to vector<3x1xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<3x1xf32> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -247,7 +247,7 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
func.func @fptrunc_f64_to_f32(%arg0: vector<4xf64>) -> vector<4xf32> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
// CHECK-NOT: nvvm
- %out = nvgpu.convert.float %arg0 : vector<4xf64> to vector<4xf32>
+ %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -257,7 +257,7 @@ func.func @fptrunc_f64_to_f16(%arg0: vector<2xf64>) -> vector<2xf16> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf16>
// CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %arg0 : vector<2xf64> to vector<2xf16>
+ %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xf16>
return %out : vector<2xf16>
}
@@ -267,7 +267,7 @@ func.func @fptrunc_f64_to_bf16(%arg0: vector<2xf64>) -> vector<2xbf16> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xbf16>
// CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %arg0 : vector<2xf64> to vector<2xbf16>
+ %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xbf16>
return %out : vector<2xbf16>
}
@@ -277,7 +277,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
// CHECK: llvm.fptrunc %{{.*}} : vector<8xf64> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.float %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
return %out : vector<4xf8E4M3FN>
}
@@ -287,7 +287,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<satfinite>}
+ %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<satfinite>}
: vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -296,7 +296,7 @@ func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.float %in {relu = true}
+ %out = nvgpu.convert.fptrunc %in {relu = true}
: vector<4xf32> to vector<4xf16>
return
}
@@ -307,7 +307,7 @@ func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.float %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -315,7 +315,7 @@ func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.float %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
return
}
@@ -323,7 +323,7 @@ func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: satfinite
- %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<none>}
+ %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<none>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -334,7 +334,7 @@ func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.float %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -343,7 +343,7 @@ func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.float %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xbf16>
return
}
@@ -352,7 +352,7 @@ func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
func.func @fptrunc_f32_to_f16_rz(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -361,7 +361,7 @@ func.func @fptrunc_f32_to_f16_rz(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_bf16_rz(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf32> to vector<4xbf16>
return
}
@@ -377,7 +377,7 @@ func.func @fptrunc_f32_to_bf16_rz(%in : vector<4xf32>) {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
- %out = nvgpu.convert.float %in : f32 to f16
+ %out = nvgpu.convert.fptrunc %in : f32 to f16
return %out : f16
}
@@ -386,7 +386,7 @@ func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
- %out = nvgpu.convert.float %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -396,6 +396,6 @@ func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f16x2.to.f8x2
// CHECK-E2E: return
func.func @e2e_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
- %out = nvgpu.convert.float %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
index 9e1798456379f..26c3afc5da358 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
@@ -2,81 +2,81 @@
// -----
-func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
- // expected-error @+1 {{'nvgpu.convert.float' op result type must be 16, 32, or 64 bitwidth, but got 8}}
- %out = nvgpu.convert.float %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
+func.func @fpext_wider(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op result type 'f4E2M1FN' must be wider than operand type 'f8E5M2'}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
return
}
// -----
-func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects bf16 or f32 output type when input type is e8m0.}}
- %out = nvgpu.convert.float %in : vector<16xf8E8M0FNU> to vector<16xf16>
+func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op result type must be 16, 32, or 64 bitwidth, but got 8}}
+ %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
return
}
// -----
-func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
- : vector<16xf8E5M2> to vector<16xf16>
+func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op expects bf16 or f32 output type when input type is e8m0.}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E8M0FNU> to vector<16xf16>
return
}
// -----
-func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.float' op relu is not supported for bf16 destination}}
- %out = nvgpu.convert.float %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
+func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.fpext %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<16xf8E5M2> to vector<16xf16>
return
}
// -----
-func.func @fpext_random_bits(%in : vector<8xf8E5M2>, %rbits : i32) {
- // expected-error @+1 {{'nvgpu.convert.float' op random_bits is only supported for truncation}}
- %out = nvgpu.convert.float %in, %rbits : vector<8xf8E5M2> to vector<8xf16>
+func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.convert.fpext' op relu is not supported for bf16 destination}}
+ %out = nvgpu.convert.fpext %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
return
}
// -----
func.func @fpext_shape_mismatch(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output shapes must match}}
- %out = nvgpu.convert.float %in : vector<16xf8E5M2> to vector<8xf16>
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output shapes must match}}
+ %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<8xf16>
return
}
// -----
func.func @fpext_scalar_vector_mismatch(%in : f8E4M3FN) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.float %in : f8E4M3FN to vector<1xf16>
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.fpext %in : f8E4M3FN to vector<1xf16>
return
}
// -----
func.func @fpext_rank0_tensor(%in : tensor<f8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.float' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.float %in : tensor<f8E4M3FN> to tensor<f16>
+ // expected-error @+1 {{'nvgpu.convert.fpext' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.fpext %in : tensor<f8E4M3FN> to tensor<f16>
return
}
// -----
func.func @fpext_container_mismatch(%in : vector<4xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.float %in : vector<4xf8E4M3FN> to tensor<4xf16>
+ // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to tensor<4xf16>
return
}
// -----
func.func @fpext_unranked_tensor(%in : tensor<*xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.float' op unranked tensor types are not supported}}
- %out = nvgpu.convert.float %in : tensor<*xf8E4M3FN> to tensor<*xf16>
+ // expected-error @+1 {{'nvgpu.convert.fpext' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.fpext %in : tensor<*xf8E4M3FN> to tensor<*xf16>
return
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
index 2ef54221e4e21..93d58a1d5dd72 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
@@ -2,25 +2,25 @@
// -----
-func.func @convert_float_same_bitwidth(%in : vector<16xf16>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output types must have different bitwidths, got 'f16' and 'bf16'}}
- %out = nvgpu.convert.float %in : vector<16xf16> to vector<16xbf16>
+func.func @fptrunc_narrower(%in : vector<16xf16>) {
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type 'f32' must be narrower than operand type 'f16'}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf16> to vector<16xf32>
return
}
// -----
func.func @fptrunc_src_bitwidth(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input type must be 64/32/16 bitwidth, but got 8}}
- %out = nvgpu.convert.float %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input type must be 64/32/16 bitwidth, but got 8}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
return
}
// -----
func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rn>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rn>}
: vector<16xf32> to vector<16xf8E8M0FNU>
return
}
@@ -28,8 +28,8 @@ func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
// -----
func.func @fptrunc_unsupported_sat_mode(%in : vector<8xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
- %out = nvgpu.convert.float %in {sat = #nvvm.sat_mode<sat>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<sat>}
: vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -37,8 +37,8 @@ func.func @fptrunc_unsupported_sat_mode(%in : vector<8xf32>) {
// -----
func.func @fptrunc_f32_to_f8_rz(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -46,8 +46,8 @@ func.func @fptrunc_f32_to_f8_rz(%in : vector<16xf32>) {
// -----
func.func @fptrunc_f64_to_f16_rz(%in : vector<4xf64>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode for f64 input, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode for f64 input, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf64> to vector<4xf16>
return
}
@@ -55,8 +55,8 @@ func.func @fptrunc_f64_to_f16_rz(%in : vector<4xf64>) {
// -----
func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -64,8 +64,8 @@ func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
// -----
func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op random_bits operand is required with RS rounding}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits operand is required with RS rounding}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -73,8 +73,8 @@ func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
// -----
func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
- %out = nvgpu.convert.float %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
+ %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -82,8 +82,8 @@ func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
// -----
func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
- // expected-error @+1 {{'nvgpu.convert.float' op random_bits can only be used with RS rounding mode}}
- %out = nvgpu.convert.float %in, %rbits
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits can only be used with RS rounding mode}}
+ %out = nvgpu.convert.fptrunc %in, %rbits
: vector<4xf32> to vector<4xf16>
return
}
@@ -91,39 +91,39 @@ func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
// -----
func.func @fptrunc_shape_mismatch(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output shapes must match}}
- %out = nvgpu.convert.float %in : vector<16xf32> to vector<8xf8E4M3FN>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output shapes must match}}
+ %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<8xf8E4M3FN>
return
}
// -----
func.func @fptrunc_scalar_vector_mismatch(%in : f32) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.float %in : f32 to vector<1xf16>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.convert.fptrunc %in : f32 to vector<1xf16>
return
}
// -----
func.func @fptrunc_rank0_tensor(%in : tensor<f32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.float %in : tensor<f32> to tensor<f16>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.convert.fptrunc %in : tensor<f32> to tensor<f16>
return
}
// -----
func.func @fptrunc_container_mismatch(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.float %in : vector<4xf32> to tensor<4xf16>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.convert.fptrunc %in : vector<4xf32> to tensor<4xf16>
return
}
// -----
func.func @fptrunc_unranked_tensor(%in : tensor<*xf32>) {
- // expected-error @+1 {{'nvgpu.convert.float' op unranked tensor types are not supported}}
- %out = nvgpu.convert.float %in : tensor<*xf32> to tensor<*xf16>
+ // expected-error @+1 {{'nvgpu.convert.fptrunc' op unranked tensor types are not supported}}
+ %out = nvgpu.convert.fptrunc %in : tensor<*xf32> to tensor<*xf16>
return
}
>From 467746b1a968238fc1771e00feb745f7a24580a2 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 3 Jul 2026 12:28:13 +0000
Subject: [PATCH 07/11] address comments
---
mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 1 -
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 37 ++++----
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 90 +++++++++----------
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 9 +-
.../nvgpu-convert-fptrunc-large.mlir | 16 ++--
.../NVGPUToNVVM/nvgpu-convert-fptrunc.mlir | 66 +++++++-------
.../NVGPU/nvgpu-convert-fptrunc-invalid.mlir | 60 ++++++-------
7 files changed, 136 insertions(+), 143 deletions(-)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index 38c8bb5c9aa2c..1c0d7bd1113ea 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -101,7 +101,6 @@ def TensorMapInterleaveKind : I32EnumAttr<"TensorMapInterleaveKind",
let cppNamespace = "::mlir::nvgpu";
}
-
def RcpApprox : I32EnumAttrCase<"APPROX", 0, "approx">;
def RcpRN : I32EnumAttrCase<"RN", 1, "rn">;
def RcpRZ : I32EnumAttrCase<"RZ", 2, "rz">;
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 9f27db4c9dcc2..85a35187eb54d 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -677,37 +677,38 @@ def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
def AnyI32Like : TypeOrValueSemanticsContainer<I32, "scalar i32 or vector of i32">;
-// nvgpu.convert.fptrunc only supports the satfinite and none saturation modes.
-def NVGPU_SaturationModeSatfiniteOrNone :
- ConfinedAttr<SaturationModeAttr, [EnumAttrIsOneOf<SaturationModeAttr,
- [SaturationModeNone, SaturationModeFinite]>]>;
-
-def NVGPU_ConvertFPTruncOp : NVGPU_Op<"convert.fptrunc", [Pure]> {
+def NVGPU_TruncfOp : NVGPU_Op<"truncf", [Pure]> {
let summary = "Truncate floating-point to narrower floating-point";
let description = [{
Truncate a floating-point value to a smaller floating-point type.
Destination must be strictly narrower than source.
Supported paths:
- f64 -> f32, f16, bf16
- f32 -> f16, bf16, f8, f6, f4
- f16 -> f8, f6, f4
- bf16 -> f8, f6, f4
+ f16 -> f8 (f8E4M3FN, f8E5M2, f8E8M0FNU), f6 (f6E2M3FN, f6E3M2FN),
+ f4 (f4E2M1FN)
+ bf16 -> f8 (f8E4M3FN, f8E5M2, f8E8M0FNU), f6 (f6E2M3FN, f6E3M2FN),
+ f4 (f4E2M1FN)
+ f32 -> f16, bf16, f8 (f8E4M3FN, f8E5M2, f8E8M0FNU),
+ f6 (f6E2M3FN, f6E3M2FN), f4 (f4E2M1FN)
+ f64 -> f32, f16, bf16
The `random_bits` operand enables stochastic rounding (RS mode) for
f32->f16/bf16 conversions; when provided, `rnd` must be RS.
Example:
```mlir
- %r = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
- %r = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
- %r = nvgpu.convert.fptrunc %in : f32 to f16
- %r = nvgpu.convert.fptrunc %in : vector<2x4xf16> to vector<2x4xf8E5M2>
+ %r = nvgpu.truncf %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %r = nvgpu.truncf %in : vector<8xf32> to vector<8xf6E2M3FN>
+ %r = nvgpu.truncf %in : f32 to f16
+ %r = nvgpu.truncf %in : vector<2x4xf16> to vector<2x4xf8E5M2>
```
}];
let arguments = (ins FloatLike:$in,
DefaultValuedAttr<FPRoundingModeAttr, "NVVM::FPRoundingMode::RN">:$rnd,
- DefaultValuedAttr<NVGPU_SaturationModeSatfiniteOrNone, "NVVM::SaturationMode::SATFINITE">:$sat,
+ DefaultValuedAttr<ConfinedAttr<SaturationModeAttr,
+ [EnumAttrIsOneOf<SaturationModeAttr,
+ [SaturationModeNone, SaturationModeFinite]>]>,
+ "NVVM::SaturationMode::SATFINITE">:$sat,
DefaultValuedAttr<BoolAttr, "false">:$relu,
Optional<I32>:$random_bits
);
@@ -723,9 +724,9 @@ def NVGPU_ConvertFPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
Destination must be strictly wider than source.
Supported paths:
- f8 -> f16, bf16
- f6 -> f16, bf16
- f4 -> f16, bf16
+ f4 (f4E2M1FN) -> f16, bf16
+ f6 (f6E2M3FN, f6E3M2FN) -> f16, bf16
+ f8 (f8E4M3FN, f8E5M2, f8E8M0FNU) -> f16, bf16
f16 -> f32
bf16 -> f32
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index abd6ac7ff8fe4..877356d5903dd 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1713,13 +1713,14 @@ struct NVGPURcpOpLowering : public ConvertOpToLLVMPattern<nvgpu::RcpOp> {
};
//===----------------------------------------------------------------------===//
-// NVGPUConvertFPTruncOp Lowering
+// NVGPUTruncfOp Lowering
//===----------------------------------------------------------------------===//
enum class FPKind { F32, BF16, F16, F8, F6, F4 };
+/// Get the effective bit width of a floating-point type.
+/// f6 types are 6-bit but NVVM Ops expect 8-bit (i8) containers.
static int getEffectiveBitWidth(int bitWidth) {
- // f6 types are 6-bit but NVVM Ops expect 8-bit (i8) containers.
return bitWidth == 6 ? 8 : bitWidth;
}
@@ -1746,15 +1747,16 @@ static std::optional<FPKind> classifyFPType(Type t) {
return FPKind::F6;
if (isConvertibleF4Type(t))
return FPKind::F4;
+
return std::nullopt;
}
/// Number of source-side i32 register slots consumed by each NVVM convert Op.
-static int getNumSrcI32PerConv(FPKind src) {
+static int getNumSrcI32PerConvert(FPKind src) {
return src == FPKind::F32 ? 2 : 1;
}
-/// Conversion op identifier for nvgpu.convert.fptrunc lowering dispatch table.
+/// Conversion op identifier for nvgpu.truncf lowering dispatch table.
enum class FPTruncConvOp {
F32x2_TO_F16x2,
F32x2_TO_BF16x2,
@@ -1792,13 +1794,16 @@ static constexpr FPTruncTableEntry kFPTruncTable[] = {
{FPKind::BF16, FPKind::F4, FPTruncConvOp::BF16x2_TO_F4x2},
};
-static std::optional<FPTruncTableEntry> lookupTruncConvOp(Type srcElemType,
- Type dstElemType) {
- auto srcKind = classifyFPType(srcElemType);
- auto dstKind = classifyFPType(dstElemType);
+/// Find the conversion table entry whose source/destination `FPKind`s match the
+/// given element types.
+template <typename TableEntry, size_t N>
+static std::optional<TableEntry>
+lookupConvOp(const TableEntry (&table)[N], Type srcElemType, Type dstElemType) {
+ std::optional<FPKind> srcKind = classifyFPType(srcElemType);
+ std::optional<FPKind> dstKind = classifyFPType(dstElemType);
if (!srcKind || !dstKind)
return std::nullopt;
- for (const auto &entry : kFPTruncTable) {
+ for (const TableEntry &entry : table) {
if (entry.src == *srcKind && entry.dst == *dstKind)
return entry;
}
@@ -1862,8 +1867,8 @@ static Value createTruncConversion(
IntegerType i8Ty = b.getI8Type();
IntegerType i16Ty = b.getI16Type();
IntegerType i32Ty = b.getI32Type();
- auto dstTyAttr = TypeAttr::get(dstElemType);
- auto actualDstTyAttr = TypeAttr::get(actualDstFloatType);
+ TypeAttr dstTyAttr = TypeAttr::get(dstElemType);
+ TypeAttr actualDstTyAttr = TypeAttr::get(actualDstFloatType);
switch (convOp) {
case FPTruncConvOp::F32x2_TO_F16x2: {
@@ -1916,10 +1921,10 @@ static Value createTruncConversion(
llvm_unreachable("unhandled FPTruncConvOp");
}
-static LogicalResult lowerFPTrunc(nvgpu::ConvertFPTruncOp op,
- nvgpu::ConvertFPTruncOp::Adaptor adaptor,
- ConversionPatternRewriter &rewriter,
- const LLVMTypeConverter *typeConverter) {
+static LogicalResult lowerTruncf(nvgpu::TruncfOp op,
+ nvgpu::TruncfOp::Adaptor adaptor,
+ ConversionPatternRewriter &rewriter,
+ const LLVMTypeConverter *typeConverter) {
MLIRContext *ctx = op.getContext();
ImplicitLocOpBuilder b(op->getLoc(), rewriter);
IntegerType i32Ty = b.getI32Type();
@@ -1944,7 +1949,7 @@ static LogicalResult lowerFPTrunc(nvgpu::ConvertFPTruncOp op,
Value randomBits = adaptor.getRandomBits();
Type actualDstFloatType = dstElemType;
- // STEP 1: bitcast input vector to i32 register vector.
+ // STEP 1: bitcast input vector to i32 vector type.
// f64 -> f32/f16/bf16 lowers to a single direct LLVM fptrunc
// f64 -> f8/f6/f4 first truncates to f32 and then reuses the narrow
// conversion path below.
@@ -1975,12 +1980,12 @@ static LogicalResult lowerFPTrunc(nvgpu::ConvertFPTruncOp op,
b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
// STEP 2: look up the conversion op from the (srcType, dstType) table.
- auto convEntry = lookupTruncConvOp(srcElemType, dstElemType);
+ auto convEntry = lookupConvOp(kFPTruncTable, srcElemType, dstElemType);
if (!convEntry)
return rewriter.notifyMatchFailure(
op, "unsupported type combination for truncation");
FPTruncConvOp convOp = convEntry->convOp;
- int numSrcI32PerConv = getNumSrcI32PerConv(convEntry->src);
+ int numSrcI32PerConv = getNumSrcI32PerConvert(convEntry->src);
// STEP 3: pack conversion results into destination i32 vector.
const int srcStep = srcBW / effectiveDstBW;
@@ -2071,19 +2076,6 @@ static constexpr FPExtTableEntry kFPExtTable[] = {
{FPKind::F4, FPKind::BF16, FPExtConvOp::F4x2_TO_BF16x2},
};
-static std::optional<FPExtTableEntry> lookupExtConvOp(Type srcElemType,
- Type dstElemType) {
- auto srcKind = classifyFPType(srcElemType);
- auto dstKind = classifyFPType(dstElemType);
- if (!srcKind || !dstKind)
- return std::nullopt;
- for (const auto &entry : kFPExtTable) {
- if (entry.src == *srcKind && entry.dst == *dstKind)
- return entry;
- }
- return std::nullopt;
-}
-
/// Create a typed NVVM extension conversion.
/// For f8/f6: src is vector<2xi8>. For f4: src is i8.
/// Returns i32 (bitcast from vector<2xf16> or vector<2xbf16>).
@@ -2199,7 +2191,7 @@ static LogicalResult lowerFPExt(nvgpu::ConvertFPExtOp op,
b.create<LLVM::UndefOp>(VectorType::get(dstI32Elems, i32Ty));
// STEP 2: look up the conversion op from the (srcType, dstType) table.
- auto convEntry = lookupExtConvOp(srcElemType, intermediateDstElem);
+ auto convEntry = lookupConvOp(kFPExtTable, srcElemType, intermediateDstElem);
if (!convEntry)
return rewriter.notifyMatchFailure(
op, "unsupported type combination for extension");
@@ -2266,17 +2258,16 @@ static LogicalResult lowerFPExt(nvgpu::ConvertFPExtOp op,
return success();
}
-struct NVGPUConvertFPTruncOpLowering
- : public ConvertOpToLLVMPattern<nvgpu::ConvertFPTruncOp> {
- using ConvertOpToLLVMPattern<nvgpu::ConvertFPTruncOp>::ConvertOpToLLVMPattern;
+struct NVGPUTruncfOpLowering : public ConvertOpToLLVMPattern<nvgpu::TruncfOp> {
+ using ConvertOpToLLVMPattern<nvgpu::TruncfOp>::ConvertOpToLLVMPattern;
LogicalResult
- matchAndRewrite(nvgpu::ConvertFPTruncOp op, OpAdaptor adaptor,
+ matchAndRewrite(nvgpu::TruncfOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
op, "tensor inputs not handled; type converter should lower first");
- return lowerFPTrunc(op, adaptor, rewriter, getTypeConverter());
+ return lowerTruncf(op, adaptor, rewriter, getTypeConverter());
}
};
@@ -2306,7 +2297,7 @@ static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
return ceilDiv(padded, step) * step;
}
-/// Canonicalization pattern for nvgpu.convert.fptrunc / nvgpu.convert.fpext:
+/// Canonicalization pattern for nvgpu.truncf / nvgpu.convert.fpext:
/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
template <typename CvtOp, bool IsTrunc>
@@ -2345,7 +2336,6 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
if (isScalar)
input = vector::BroadcastOp::create(b, VectorType::get({1}, srcElemTy),
input);
-
if (isMultiRank)
input = vector::ShapeCastOp::create(
b, VectorType::get({numElems}, srcElemTy), input);
@@ -2361,33 +2351,37 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
auto cvtDstTy =
VectorType::get({needsPad ? paddedElems : numElems}, dstElemTy);
Value cvt;
- if constexpr (IsTrunc)
+ if constexpr (IsTrunc) {
cvt = CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getSatAttr(),
op.getReluAttr(), op.getRandomBits());
- else
+ } else {
cvt =
CvtOp::create(b, cvtDstTy, input, op.getRndAttr(), op.getReluAttr());
+ }
Value result = cvt;
- if (needsPad)
+ if (needsPad) {
result = vector::ExtractStridedSliceOp::create(
b, result, SmallVector<int64_t>{0}, SmallVector<int64_t>{numElems},
SmallVector<int64_t>{1});
+ }
- if (isMultiRank)
+ if (isMultiRank) {
result =
vector::ShapeCastOp::create(b, cast<VectorType>(outType), result);
+ }
- if (isScalar)
+ if (isScalar) {
result = vector::ExtractOp::create(b, result, SmallVector<int64_t>{0});
+ }
rewriter.replaceOp(op, result);
return success();
}
};
-using NVGPUConvertFPTruncCanonicalizePattern =
- NVGPUFPCanonicalizePattern<nvgpu::ConvertFPTruncOp, true>;
+using NVGPUTruncfCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::TruncfOp, true>;
using NVGPUConvertFPExtCanonicalizePattern =
NVGPUFPCanonicalizePattern<nvgpu::ConvertFPExtOp, false>;
} // namespace
@@ -2434,12 +2428,12 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUWarpgroupMmaOpLowering, // nvgpu.warpgroup.mma
NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
- NVGPUConvertFPTruncOpLowering, // nvgpu.convert.fptrunc
+ NVGPUTruncfOpLowering, // nvgpu.truncf
NVGPUConvertFPExtOpLowering, // nvgpu.convert.fpext
MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
- patterns.add<NVGPUConvertFPTruncCanonicalizePattern,
+ patterns.add<NVGPUTruncfCanonicalizePattern,
NVGPUConvertFPExtCanonicalizePattern>(patterns.getContext());
}
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 98973a733f063..607888b678df1 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -701,7 +701,7 @@ LogicalResult RcpOp::verify() {
}
//===----------------------------------------------------------------------===//
-// NVGPU_ConvertFPTruncOp
+// NVGPU_TruncfOp
//===----------------------------------------------------------------------===//
static bool isShapedContainerType(Type t) {
@@ -737,7 +737,7 @@ static LogicalResult verifyConversionShapes(Operation *op, Type inType,
return success();
}
-LogicalResult ConvertFPTruncOp::verify() {
+LogicalResult TruncfOp::verify() {
Type inType = getIn().getType();
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
@@ -806,9 +806,8 @@ LogicalResult ConvertFPExtOp::verify() {
int dstBitWidth = dstType.getIntOrFloatBitWidth();
auto rnd = getRnd();
- if (auto result = verifyConversionShapes(getOperation(), inType, outType);
- failed(result))
- return result;
+ if (failed(verifyConversionShapes(getOperation(), inType, outType)))
+ return failure();
if (srcBitWidth >= dstBitWidth)
return emitOpError("result type ")
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
index d6c862fa5a7a6..c5a66ccc3751f 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.fptrunc.
+// Large-vector smoke tests for nvgpu.truncf.
// CHECK-LABEL: @cvt_large_f32_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf32>
@@ -10,7 +10,7 @@ func.func @cvt_large_f32_to_f16(%in : vector<400xf32>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf16>
+ %out = nvgpu.truncf %in : vector<400xf32> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_f32_to_bf16(%in : vector<400xf32>) -> vector<400xbf16> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xbf16>
+ %out = nvgpu.truncf %in : vector<400xf32> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -31,7 +31,7 @@ func.func @cvt_large_f32_to_f8(%in : vector<400xf32>) -> vector<400xf8E4M3FN> {
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f8x2
// CHECK-NOT: nvvm.convert.f32x2.to.f8x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<400xf32> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -42,7 +42,7 @@ func.func @cvt_large_f32_to_f6(%in : vector<400xf32>) -> vector<400xf6E2M3FN> {
// CHECK-NOT: nvvm.convert.f32x2.to.f6x2
// CHECK: llvm.bitcast {{.*}} : vector<100xi32> to vector<400xi8>
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf6E2M3FN>
+ %out = nvgpu.truncf %in : vector<400xf32> to vector<400xf6E2M3FN>
return %out : vector<400xf6E2M3FN>
}
@@ -52,7 +52,7 @@ func.func @cvt_large_f32_to_f4(%in : vector<400xf32>) -> vector<400xf4E2M1FN> {
// CHECK: llvm.mlir.undef : vector<50xi32>
// CHECK-COUNT-200: nvvm.convert.f32x2.to.f4x2
// CHECK-NOT: nvvm.convert.f32x2.to.f4x2
- %out = nvgpu.convert.fptrunc %in : vector<400xf32> to vector<400xf4E2M1FN>
+ %out = nvgpu.truncf %in : vector<400xf32> to vector<400xf4E2M1FN>
return %out : vector<400xf4E2M1FN>
}
@@ -61,7 +61,7 @@ func.func @cvt_large_f16_to_f8(%in : vector<400xf16>) -> vector<400xf8E4M3FN> {
// CHECK: llvm.bitcast %{{.*}} : vector<400xf16> to vector<200xi32>
// CHECK-COUNT-200: nvvm.convert.f16x2.to.f8x2
// CHECK-NOT: nvvm.convert.f16x2.to.f8x2
- %out = nvgpu.convert.fptrunc %in : vector<400xf16> to vector<400xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<400xf16> to vector<400xf8E4M3FN>
return %out : vector<400xf8E4M3FN>
}
@@ -71,6 +71,6 @@ func.func @cvt_large_bf16_to_f6(%in : vector<400xbf16>) -> vector<400xf6E3M2FN>
// CHECK-COUNT-200: nvvm.convert.bf16x2.to.f6x2
// CHECK-NOT: nvvm.convert.bf16x2.to.f6x2
// CHECK: llvm.trunc {{.*}} : vector<400xi8> to vector<400xi6>
- %out = nvgpu.convert.fptrunc %in : vector<400xbf16> to vector<400xf6E3M2FN>
+ %out = nvgpu.truncf %in : vector<400xbf16> to vector<400xf6E3M2FN>
return %out : vector<400xf6E3M2FN>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
index 8d1c71e22f366..d34402fcf1ce0 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
@@ -14,7 +14,7 @@ func.func @cvt_float_f32_to_f16(%in : vector<4xf32>) {
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rn>
// CHECK-SAME: : vector<2xf16>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.truncf %in : vector<4xf32> to vector<4xf16>
return
}
@@ -27,7 +27,7 @@ func.func @cvt_float_f32_to_f16_v8(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf16>
+ %out = nvgpu.truncf %in : vector<8xf32> to vector<8xf16>
return
}
@@ -40,7 +40,7 @@ func.func @cvt_float_f32_to_bf16(%in : vector<4xf32>) {
// CHECK-SAME: : vector<2xbf16>
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xbf16>
+ %out = nvgpu.truncf %in : vector<4xf32> to vector<4xbf16>
return
}
@@ -66,7 +66,7 @@ func.func @cvt_float_f32_to_e4m3(%in : vector<8xf32>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -85,7 +85,7 @@ func.func @cvt_float_f16_to_e2m3(%in : vector<8xf16>) {
// CHECK: llvm.bitcast {{.*}} : vector<2xi16> to i32
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xf16> to vector<8xf6E2M3FN>
+ %out = nvgpu.truncf %in : vector<8xf16> to vector<8xf6E2M3FN>
return
}
@@ -99,7 +99,7 @@ func.func @cvt_float_bf16_to_e3m2(%in : vector<8xbf16>) {
// CHECK-SAME: : vector<2xbf16> -> i16(f6E3M2FN)
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xbf16> to vector<8xf6E3M2FN>
+ %out = nvgpu.truncf %in : vector<8xbf16> to vector<8xf6E3M2FN>
return
}
@@ -124,7 +124,7 @@ func.func @cvt_float_f32_to_e2m3(%in : vector<8xf32>) {
// CHECK: llvm.insertelement {{.*}} : vector<2xi32>
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<8xi8>
// CHECK: llvm.trunc {{.*}} : vector<8xi8> to vector<8xi6>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf6E2M3FN>
+ %out = nvgpu.truncf %in : vector<8xf32> to vector<8xf6E2M3FN>
return
}
@@ -134,7 +134,7 @@ func.func @cvt_float_f32_to_e8m0_rz(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
// CHECK-SAME: : i16(f8E8M0FNU)
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<8xf32> to vector<8xf8E8M0FNU>
return
}
@@ -145,7 +145,7 @@ func.func @cvt_float_f32_to_e8m0_rp(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rp>
// CHECK-SAME: : i16(f8E8M0FNU)
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rp>}
: vector<8xf32> to vector<8xf8E8M0FNU>
return
}
@@ -160,7 +160,7 @@ func.func @fptrunc_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.fptrunc %in : f32 to f16
+ %out = nvgpu.truncf %in : f32 to f16
return %out : f16
}
@@ -170,7 +170,7 @@ func.func @fptrunc_scalar_f32_to_bf16(%in : f32) -> bf16 {
// CHECK: vector.broadcast %[[IN]]
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK: vector.extract
- %out = nvgpu.convert.fptrunc %in : f32 to bf16
+ %out = nvgpu.truncf %in : f32 to bf16
return %out : bf16
}
@@ -179,7 +179,7 @@ func.func @fptrunc_scalar_f64_to_f32(%arg0: f64) -> f32 {
// CHECK: vector.broadcast
// CHECK: llvm.fptrunc
// CHECK: vector.extract
- %out = nvgpu.convert.fptrunc %arg0 : f64 to f32
+ %out = nvgpu.truncf %arg0 : f64 to f32
return %out : f32
}
@@ -191,7 +191,7 @@ func.func @fptrunc_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.truncf %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -201,7 +201,7 @@ func.func @fptrunc_v4x2_f32_to_f8(%in : vector<4x2xf32>) -> vector<4x2xf8E4M3FN>
// CHECK: vector.shape_cast %[[IN]] : vector<4x2xf32> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.shape_cast {{.*}} to vector<4x2xf8E4M3FN>
- %out = nvgpu.convert.fptrunc %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<4x2xf32> to vector<4x2xf8E4M3FN>
return %out : vector<4x2xf8E4M3FN>
}
@@ -213,7 +213,7 @@ func.func @fptrunc_v1f32_to_v1f16(%in : vector<1xf32>) -> vector<1xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %in : vector<1xf32> to vector<1xf16>
+ %out = nvgpu.truncf %in : vector<1xf32> to vector<1xf16>
return %out : vector<1xf16>
}
@@ -223,7 +223,7 @@ func.func @fptrunc_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f16x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
@@ -237,7 +237,7 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.fptrunc %in : vector<3x1xf32> to vector<3x1xf16>
+ %out = nvgpu.truncf %in : vector<3x1xf32> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -247,7 +247,7 @@ func.func @fptrunc_v3x1_f32_to_f16(%in : vector<3x1xf32>) -> vector<3x1xf16> {
func.func @fptrunc_f64_to_f32(%arg0: vector<4xf64>) -> vector<4xf32> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf32>
// CHECK-NOT: nvvm
- %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf32>
+ %out = nvgpu.truncf %arg0 : vector<4xf64> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -257,7 +257,7 @@ func.func @fptrunc_f64_to_f16(%arg0: vector<2xf64>) -> vector<2xf16> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xf16>
// CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xf16>
+ %out = nvgpu.truncf %arg0 : vector<2xf64> to vector<2xf16>
return %out : vector<2xf16>
}
@@ -267,7 +267,7 @@ func.func @fptrunc_f64_to_bf16(%arg0: vector<2xf64>) -> vector<2xbf16> {
// CHECK: llvm.fptrunc %{{.*}} : vector<4xf64> to vector<4xbf16>
// CHECK-NOT: nvvm.convert
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<2xf64> to vector<2xbf16>
+ %out = nvgpu.truncf %arg0 : vector<2xf64> to vector<2xbf16>
return %out : vector<2xbf16>
}
@@ -277,7 +277,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
// CHECK: llvm.fptrunc %{{.*}} : vector<8xf64> to vector<8xf32>
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fptrunc %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
+ %out = nvgpu.truncf %arg0 : vector<4xf64> to vector<4xf8E4M3FN>
return %out : vector<4xf8E4M3FN>
}
@@ -287,7 +287,7 @@ func.func @fptrunc_f64_to_f8(%arg0: vector<4xf64>) -> vector<4xf8E4M3FN> {
func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<satfinite>}
+ %out = nvgpu.truncf %in {sat = #nvvm.sat_mode<satfinite>}
: vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -296,7 +296,7 @@ func.func @fptrunc_f32_to_f8_satfinite(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.fptrunc %in {relu = true}
+ %out = nvgpu.truncf %in {relu = true}
: vector<4xf32> to vector<4xf16>
return
}
@@ -307,7 +307,7 @@ func.func @fptrunc_f32_to_f16_relu(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
// CHECK: nvvm.convert.f32x2.to.f8x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in : vector<8xf32> to vector<8xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -315,7 +315,7 @@ func.func @fptrunc_f32_to_f8_default_sat(%in : vector<8xf32>) {
func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: sat = #nvvm.sat_mode<satfinite>
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to vector<4xf16>
+ %out = nvgpu.truncf %in : vector<4xf32> to vector<4xf16>
return
}
@@ -323,7 +323,7 @@ func.func @fptrunc_f32_to_f16_default_sat(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-NOT: satfinite
- %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<none>}
+ %out = nvgpu.truncf %in {sat = #nvvm.sat_mode<none>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -334,7 +334,7 @@ func.func @fptrunc_f32_to_f16_explicit_none(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.truncf %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -343,7 +343,7 @@ func.func @fptrunc_f32_to_f16_rs(%in : vector<4xf32>, %rbits : i32) {
func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rs>
- %out = nvgpu.convert.fptrunc %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
+ %out = nvgpu.truncf %in, %rbits {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xbf16>
return
}
@@ -352,7 +352,7 @@ func.func @fptrunc_f32_to_bf16_rs(%in : vector<4xf32>, %rbits : i32) {
func.func @fptrunc_f32_to_f16_rz(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.f16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -361,7 +361,7 @@ func.func @fptrunc_f32_to_f16_rz(%in : vector<4xf32>) {
func.func @fptrunc_f32_to_bf16_rz(%in : vector<4xf32>) {
// CHECK: nvvm.convert.f32x2.to.bf16x2
// CHECK-SAME: rnd = #nvvm.fp_rnd_mode<rz>
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf32> to vector<4xbf16>
return
}
@@ -377,7 +377,7 @@ func.func @fptrunc_f32_to_bf16_rz(%in : vector<4xf32>) {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
- %out = nvgpu.convert.fptrunc %in : f32 to f16
+ %out = nvgpu.truncf %in : f32 to f16
return %out : f16
}
@@ -386,7 +386,7 @@ func.func @e2e_scalar_f32_to_f16(%in : f32) -> f16 {
// CHECK-E2E: nvvm.convert.f32x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
- %out = nvgpu.convert.fptrunc %in : vector<2x4xf32> to vector<2x4xf16>
+ %out = nvgpu.truncf %in : vector<2x4xf32> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -396,6 +396,6 @@ func.func @e2e_v2x4_f32_to_f16(%in : vector<2x4xf32>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f16x2.to.f8x2
// CHECK-E2E: return
func.func @e2e_v3f16_to_v3f8(%in : vector<3xf16>) -> vector<3xf8E4M3FN> {
- %out = nvgpu.convert.fptrunc %in : vector<3xf16> to vector<3xf8E4M3FN>
+ %out = nvgpu.truncf %in : vector<3xf16> to vector<3xf8E4M3FN>
return %out : vector<3xf8E4M3FN>
}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
index 93d58a1d5dd72..9cee9720bddda 100644
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
@@ -3,24 +3,24 @@
// -----
func.func @fptrunc_narrower(%in : vector<16xf16>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op result type 'f32' must be narrower than operand type 'f16'}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf16> to vector<16xf32>
+ // expected-error @+1 {{'nvgpu.truncf' op result type 'f32' must be narrower than operand type 'f16'}}
+ %out = nvgpu.truncf %in : vector<16xf16> to vector<16xf32>
return
}
// -----
func.func @fptrunc_src_bitwidth(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input type must be 64/32/16 bitwidth, but got 8}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ // expected-error @+1 {{'nvgpu.truncf' op input type must be 64/32/16 bitwidth, but got 8}}
+ %out = nvgpu.truncf %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
return
}
// -----
func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rn>}
+ // expected-error @+1 {{'nvgpu.truncf' op expects RZ or RP rounding mode when result type is e8m0, but got #nvvm.fp_rnd_mode<rn>}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rn>}
: vector<16xf32> to vector<16xf8E8M0FNU>
return
}
@@ -28,8 +28,8 @@ func.func @fptrunc_e8m0_bad_rounding(%in : vector<16xf32>) {
// -----
func.func @fptrunc_unsupported_sat_mode(%in : vector<8xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
- %out = nvgpu.convert.fptrunc %in {sat = #nvvm.sat_mode<sat>}
+ // expected-error @+1 {{'nvgpu.truncf' op attribute 'sat' failed to satisfy constraint: Describes the saturation mode whose value is one of {none, satfinite}}}
+ %out = nvgpu.truncf %in {sat = #nvvm.sat_mode<sat>}
: vector<8xf32> to vector<8xf8E4M3FN>
return
}
@@ -37,8 +37,8 @@ func.func @fptrunc_unsupported_sat_mode(%in : vector<8xf32>) {
// -----
func.func @fptrunc_f32_to_f8_rz(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ // expected-error @+1 {{'nvgpu.truncf' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -46,8 +46,8 @@ func.func @fptrunc_f32_to_f8_rz(%in : vector<16xf32>) {
// -----
func.func @fptrunc_f64_to_f16_rz(%in : vector<4xf64>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode for f64 input, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ // expected-error @+1 {{'nvgpu.truncf' op expects RN rounding mode for f64 input, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rz>}
: vector<4xf64> to vector<4xf16>
return
}
@@ -55,8 +55,8 @@ func.func @fptrunc_f64_to_f16_rz(%in : vector<4xf64>) {
// -----
func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.truncf' op RS (stochastic) rounding is only supported for f32->f16/bf16, got 'f32' -> 'f8E4M3FN'}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -64,8 +64,8 @@ func.func @fptrunc_rs_unsupported_types(%in : vector<16xf32>) {
// -----
func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits operand is required with RS rounding}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rs>}
+ // expected-error @+1 {{'nvgpu.truncf' op random_bits operand is required with RS rounding}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rs>}
: vector<4xf32> to vector<4xf16>
return
}
@@ -73,8 +73,8 @@ func.func @fptrunc_rs_no_random_bits(%in : vector<4xf32>) {
// -----
func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
- %out = nvgpu.convert.fptrunc %in {rnd = #nvvm.fp_rnd_mode<rp>}
+ // expected-error @+1 {{'nvgpu.truncf' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rp>}}
+ %out = nvgpu.truncf %in {rnd = #nvvm.fp_rnd_mode<rp>}
: vector<16xf32> to vector<16xf8E4M3FN>
return
}
@@ -82,8 +82,8 @@ func.func @fptrunc_bad_rounding(%in : vector<16xf32>) {
// -----
func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op random_bits can only be used with RS rounding mode}}
- %out = nvgpu.convert.fptrunc %in, %rbits
+ // expected-error @+1 {{'nvgpu.truncf' op random_bits can only be used with RS rounding mode}}
+ %out = nvgpu.truncf %in, %rbits
: vector<4xf32> to vector<4xf16>
return
}
@@ -91,39 +91,39 @@ func.func @fptrunc_random_bits_no_rs(%in : vector<4xf32>, %rbits : i32) {
// -----
func.func @fptrunc_shape_mismatch(%in : vector<16xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output shapes must match}}
- %out = nvgpu.convert.fptrunc %in : vector<16xf32> to vector<8xf8E4M3FN>
+ // expected-error @+1 {{'nvgpu.truncf' op input and output shapes must match}}
+ %out = nvgpu.truncf %in : vector<16xf32> to vector<8xf8E4M3FN>
return
}
// -----
func.func @fptrunc_scalar_vector_mismatch(%in : f32) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.fptrunc %in : f32 to vector<1xf16>
+ // expected-error @+1 {{'nvgpu.truncf' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.truncf %in : f32 to vector<1xf16>
return
}
// -----
func.func @fptrunc_rank0_tensor(%in : tensor<f32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.fptrunc %in : tensor<f32> to tensor<f16>
+ // expected-error @+1 {{'nvgpu.truncf' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.truncf %in : tensor<f32> to tensor<f16>
return
}
// -----
func.func @fptrunc_container_mismatch(%in : vector<4xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.fptrunc %in : vector<4xf32> to tensor<4xf16>
+ // expected-error @+1 {{'nvgpu.truncf' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.truncf %in : vector<4xf32> to tensor<4xf16>
return
}
// -----
func.func @fptrunc_unranked_tensor(%in : tensor<*xf32>) {
- // expected-error @+1 {{'nvgpu.convert.fptrunc' op unranked tensor types are not supported}}
- %out = nvgpu.convert.fptrunc %in : tensor<*xf32> to tensor<*xf16>
+ // expected-error @+1 {{'nvgpu.truncf' op unranked tensor types are not supported}}
+ %out = nvgpu.truncf %in : tensor<*xf32> to tensor<*xf16>
return
}
>From bd1e5df96af41323971ac778226419735aee779b Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 3 Jul 2026 12:38:22 +0000
Subject: [PATCH 08/11] rename convert.fpext to extf and rename test files
---
.../include/mlir/Dialect/NVGPU/IR/NVGPUOps.td | 10 +--
.../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 28 +++----
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 4 +-
...fpext-large.mlir => nvgpu-extf-large.mlir} | 16 ++--
...gpu-convert-fpext.mlir => nvgpu-extf.mlir} | 70 ++++++++--------
...unc-large.mlir => nvgpu-truncf-large.mlir} | 0
...convert-fptrunc.mlir => nvgpu-truncf.mlir} | 0
.../NVGPU/nvgpu-convert-fpext-invalid.mlir | 82 -------------------
.../Dialect/NVGPU/nvgpu-extf-invalid.mlir | 82 +++++++++++++++++++
...invalid.mlir => nvgpu-truncf-invalid.mlir} | 0
10 files changed, 146 insertions(+), 146 deletions(-)
rename mlir/test/Conversion/NVGPUToNVVM/{nvgpu-convert-fpext-large.mlir => nvgpu-extf-large.mlir} (84%)
rename mlir/test/Conversion/NVGPUToNVVM/{nvgpu-convert-fpext.mlir => nvgpu-extf.mlir} (87%)
rename mlir/test/Conversion/NVGPUToNVVM/{nvgpu-convert-fptrunc-large.mlir => nvgpu-truncf-large.mlir} (100%)
rename mlir/test/Conversion/NVGPUToNVVM/{nvgpu-convert-fptrunc.mlir => nvgpu-truncf.mlir} (100%)
delete mode 100644 mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
create mode 100644 mlir/test/Dialect/NVGPU/nvgpu-extf-invalid.mlir
rename mlir/test/Dialect/NVGPU/{nvgpu-convert-fptrunc-invalid.mlir => nvgpu-truncf-invalid.mlir} (100%)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
index 85a35187eb54d..88ffc69c31cb2 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -717,7 +717,7 @@ def NVGPU_TruncfOp : NVGPU_Op<"truncf", [Pure]> {
let hasVerifier = 1;
}
-def NVGPU_ConvertFPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
+def NVGPU_ExtfOp : NVGPU_Op<"extf", [Pure]> {
let summary = "Extend floating-point to wider floating-point";
let description = [{
Extend a floating-point value to a wider floating-point type.
@@ -732,10 +732,10 @@ def NVGPU_ConvertFPExtOp : NVGPU_Op<"convert.fpext", [Pure]> {
Example:
```mlir
- %r = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
- %r = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
- %r = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
- %r = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ %r = nvgpu.extf %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %r = nvgpu.extf %in : vector<8xf6E3M2FN> to vector<8xf16>
+ %r = nvgpu.extf %in : vector<4xf8E5M2> to vector<4xf32>
+ %r = nvgpu.extf %in : f8E4M3FN to f32
```
}];
let arguments = (ins FloatLike:$in,
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 877356d5903dd..bb0bbc2d0a284 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -2048,10 +2048,10 @@ static LogicalResult lowerTruncf(nvgpu::TruncfOp op,
}
//===----------------------------------------------------------------------===//
-// NVGPUConvertFPExtOp Lowering
+// NVGPUExtfOp Lowering
//===----------------------------------------------------------------------===//
-/// Conversion op identifier for nvgpu.convert.fpext lowering dispatch table.
+/// Conversion op identifier for nvgpu.extf lowering dispatch table.
enum class FPExtConvOp {
F8x2_TO_F16x2,
F8x2_TO_BF16x2,
@@ -2121,8 +2121,8 @@ static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
llvm_unreachable("unhandled FPExtConvOp");
}
-static LogicalResult lowerFPExt(nvgpu::ConvertFPExtOp op,
- nvgpu::ConvertFPExtOp::Adaptor adaptor,
+static LogicalResult lowerExtf(nvgpu::ExtfOp op,
+ nvgpu::ExtfOp::Adaptor adaptor,
ConversionPatternRewriter &rewriter,
const LLVMTypeConverter *typeConverter) {
MLIRContext *ctx = op.getContext();
@@ -2271,17 +2271,17 @@ struct NVGPUTruncfOpLowering : public ConvertOpToLLVMPattern<nvgpu::TruncfOp> {
}
};
-struct NVGPUConvertFPExtOpLowering
- : public ConvertOpToLLVMPattern<nvgpu::ConvertFPExtOp> {
- using ConvertOpToLLVMPattern<nvgpu::ConvertFPExtOp>::ConvertOpToLLVMPattern;
+struct NVGPUExtfOpLowering
+ : public ConvertOpToLLVMPattern<nvgpu::ExtfOp> {
+ using ConvertOpToLLVMPattern<nvgpu::ExtfOp>::ConvertOpToLLVMPattern;
LogicalResult
- matchAndRewrite(nvgpu::ConvertFPExtOp op, OpAdaptor adaptor,
+ matchAndRewrite(nvgpu::ExtfOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
if (isa<RankedTensorType>(op.getIn().getType()))
return rewriter.notifyMatchFailure(
op, "tensor inputs not handled; type converter should lower first");
- return lowerFPExt(op, adaptor, rewriter, getTypeConverter());
+ return lowerExtf(op, adaptor, rewriter, getTypeConverter());
}
};
@@ -2297,7 +2297,7 @@ static int64_t computePaddedElems(int64_t numElems, int srcBW, int dstBW,
return ceilDiv(padded, step) * step;
}
-/// Canonicalization pattern for nvgpu.truncf / nvgpu.convert.fpext:
+/// Canonicalization pattern for nvgpu.truncf / nvgpu.extf:
/// handles scalar inputs, non-32-bit-aligned vectors, and multi-rank vectors.
/// Runs as an OpRewritePattern on MLIR types before LLVM type conversion.
template <typename CvtOp, bool IsTrunc>
@@ -2382,8 +2382,8 @@ struct NVGPUFPCanonicalizePattern : public OpRewritePattern<CvtOp> {
using NVGPUTruncfCanonicalizePattern =
NVGPUFPCanonicalizePattern<nvgpu::TruncfOp, true>;
-using NVGPUConvertFPExtCanonicalizePattern =
- NVGPUFPCanonicalizePattern<nvgpu::ConvertFPExtOp, false>;
+using NVGPUExtfCanonicalizePattern =
+ NVGPUFPCanonicalizePattern<nvgpu::ExtfOp, false>;
} // namespace
void mlir::nvgpu::populateCommonGPUTypeAndAttributeConversions(
@@ -2429,11 +2429,11 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUWarpgroupMmaStoreOpLowering, // nvgpu.warpgroup.mma.store
NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
NVGPUTruncfOpLowering, // nvgpu.truncf
- NVGPUConvertFPExtOpLowering, // nvgpu.convert.fpext
+ NVGPUExtfOpLowering, // nvgpu.extf
MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
patterns.add<NVGPUTruncfCanonicalizePattern,
- NVGPUConvertFPExtCanonicalizePattern>(patterns.getContext());
+ NVGPUExtfCanonicalizePattern>(patterns.getContext());
}
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 607888b678df1..8dd67411292dd 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -794,10 +794,10 @@ LogicalResult TruncfOp::verify() {
}
//===----------------------------------------------------------------------===//
-// NVGPU_ConvertFPExtOp
+// NVGPU_ExtfOp
//===----------------------------------------------------------------------===//
-LogicalResult ConvertFPExtOp::verify() {
+LogicalResult ExtfOp::verify() {
Type inType = getIn().getType();
Type outType = getType();
Type srcType = getElementTypeOrSelf(inType);
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf-large.mlir
similarity index 84%
rename from mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
rename to mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf-large.mlir
index 35379861f8f5d..9f2ddc5270c50 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext-large.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf-large.mlir
@@ -1,6 +1,6 @@
// RUN: mlir-opt %s -convert-nvgpu-to-nvvm | FileCheck %s
-// Large-vector smoke tests for nvgpu.convert.fpext
+// Large-vector smoke tests for nvgpu.extf
// CHECK-LABEL: @cvt_large_f8_to_f16(
// CHECK-SAME: %[[IN:.+]]: vector<400xf8E4M3FN>
@@ -10,7 +10,7 @@ func.func @cvt_large_f8_to_f16(%in : vector<400xf8E4M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f8x2.to.f16x2
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf8E4M3FN> to vector<400xf16>
+ %out = nvgpu.extf %in : vector<400xf8E4M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -20,7 +20,7 @@ func.func @cvt_large_e8m0_to_bf16(%in : vector<400xf8E8M0FNU>) -> vector<400xbf1
// CHECK-COUNT-200: nvvm.convert.f8x2.to.bf16x2
// CHECK-NOT: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xbf16>
- %out = nvgpu.convert.fpext %in : vector<400xf8E8M0FNU> to vector<400xbf16>
+ %out = nvgpu.extf %in : vector<400xf8E8M0FNU> to vector<400xbf16>
return %out : vector<400xbf16>
}
@@ -33,7 +33,7 @@ func.func @cvt_large_f6_to_f16(%in : vector<400xf6E2M3FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf16>
+ %out = nvgpu.extf %in : vector<400xf6E2M3FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -44,7 +44,7 @@ func.func @cvt_large_f4_to_f16(%in : vector<400xf4E2M1FN>) -> vector<400xf16> {
// CHECK-COUNT-200: nvvm.convert.f4x2.to.f16x2
// CHECK-NOT: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
- %out = nvgpu.convert.fpext %in : vector<400xf4E2M1FN> to vector<400xf16>
+ %out = nvgpu.extf %in : vector<400xf4E2M1FN> to vector<400xf16>
return %out : vector<400xf16>
}
@@ -54,7 +54,7 @@ func.func @cvt_large_f8_to_f32(%in : vector<400xf8E5M2>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<200xi32> to vector<400xf16>
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf8E5M2> to vector<400xf32>
+ %out = nvgpu.extf %in : vector<400xf8E5M2> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -64,7 +64,7 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
// CHECK-COUNT-200: nvvm.convert.f6x2.to.f16x2
// CHECK-NOT: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf6E2M3FN> to vector<400xf32>
+ %out = nvgpu.extf %in : vector<400xf6E2M3FN> to vector<400xf32>
return %out : vector<400xf32>
}
@@ -73,6 +73,6 @@ func.func @cvt_large_f6_to_f32(%in : vector<400xf6E2M3FN>) -> vector<400xf32> {
func.func @cvt_large_f16_to_f32(%in : vector<400xf16>) -> vector<400xf32> {
// CHECK-NOT: nvvm.convert
// CHECK: llvm.fpext %[[IN]] : vector<400xf16> to vector<400xf32>
- %out = nvgpu.convert.fpext %in : vector<400xf16> to vector<400xf32>
+ %out = nvgpu.extf %in : vector<400xf16> to vector<400xf32>
return %out : vector<400xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf.mlir
similarity index 87%
rename from mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
rename to mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf.mlir
index 1d711540f7aff..5924f984f870d 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fpext.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-extf.mlir
@@ -18,7 +18,7 @@ func.func @cvt_float_e4m3fn_to_f16(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E4M3FN> to vector<8xf16>
+ %out = nvgpu.extf %in : vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -48,7 +48,7 @@ func.func @cvt_float_e5m2_to_f16(%in : vector<8xf8E5M2>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E5M2> to vector<8xf16>
+ %out = nvgpu.extf %in : vector<8xf8E5M2> to vector<8xf16>
return
}
@@ -70,7 +70,7 @@ func.func @cvt_float_e8m0_to_bf16(%in : vector<8xf8E8M0FNU>) {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
- %out = nvgpu.convert.fpext %in : vector<8xf8E8M0FNU> to vector<8xbf16>
+ %out = nvgpu.extf %in : vector<8xf8E8M0FNU> to vector<8xbf16>
return
}
@@ -92,7 +92,7 @@ func.func @cvt_float_e2m3_to_f16(%in : vector<8xf6E2M3FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xf16>
+ %out = nvgpu.extf %in : vector<8xf6E2M3FN> to vector<8xf16>
return
}
@@ -117,7 +117,7 @@ func.func @cvt_float_e3m2_to_f16(%in : vector<8xf6E3M2FN>) {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
- %out = nvgpu.convert.fpext %in : vector<8xf6E3M2FN> to vector<8xf16>
+ %out = nvgpu.extf %in : vector<8xf6E3M2FN> to vector<8xf16>
return
}
@@ -150,7 +150,7 @@ func.func @cvt_float_e2m1_to_f16(%in : vector<16xf4E2M1FN>) {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xf16>
- %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf16>
+ %out = nvgpu.extf %in : vector<16xf4E2M1FN> to vector<16xf16>
return
}
@@ -168,7 +168,7 @@ func.func @cvt_float_e2m3_to_bf16(%in : vector<8xf6E2M3FN>) {
// CHECK: nvvm.convert.f6x2.to.bf16x2
// CHECK: nvvm.convert.f6x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xbf16>
- %out = nvgpu.convert.fpext %in : vector<8xf6E2M3FN> to vector<8xbf16>
+ %out = nvgpu.extf %in : vector<8xf6E2M3FN> to vector<8xbf16>
return
}
@@ -183,7 +183,7 @@ func.func @cvt_float_e2m1_to_bf16(%in : vector<16xf4E2M1FN>) {
// CHECK-SAME: : i8(f4E2M1FN)
// CHECK: nvvm.convert.f4x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<8xi32> to vector<16xbf16>
- %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xbf16>
+ %out = nvgpu.extf %in : vector<16xf4E2M1FN> to vector<16xbf16>
return
}
@@ -200,7 +200,7 @@ func.func @fpext_e4m3fn_to_f32(%in : vector<4xf8E4M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xf8E4M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -213,7 +213,7 @@ func.func @fpext_e5m2_to_f32(%in : vector<4xf8E5M2>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E5M2> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xf8E5M2> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -226,7 +226,7 @@ func.func @fpext_e8m0_to_f32(%in : vector<4xf8E8M0FNU>) -> vector<4xf32> {
// CHECK: nvvm.convert.f8x2.to.bf16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xbf16>
// CHECK: llvm.fpext {{.*}} : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf8E8M0FNU> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xf8E8M0FNU> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -241,7 +241,7 @@ func.func @fpext_e2m3_to_f32(%in : vector<4xf6E2M3FN>) -> vector<4xf32> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<2xi32> to vector<4xf16>
// CHECK: llvm.fpext {{.*}} : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf6E2M3FN> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xf6E2M3FN> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -253,7 +253,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.bitcast {{.*}} : vector<4xi32> to vector<8xf16>
// CHECK: llvm.fpext {{.*}} : vector<8xf16> to vector<8xf32>
- %out = nvgpu.convert.fpext %in : vector<8xf4E2M1FN> to vector<8xf32>
+ %out = nvgpu.extf %in : vector<8xf4E2M1FN> to vector<8xf32>
return %out : vector<8xf32>
}
@@ -263,7 +263,7 @@ func.func @fpext_e2m1_to_f32(%in : vector<8xf4E2M1FN>) -> vector<8xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xf16>
func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xf16> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -273,7 +273,7 @@ func.func @fpext_f16_to_f32(%in : vector<4xf16>) -> vector<4xf32> {
// CHECK-SAME: %[[IN:.+]]: vector<4xbf16>
func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
// CHECK: llvm.fpext %[[IN]] : vector<4xbf16> to vector<4xf32>
- %out = nvgpu.convert.fpext %in : vector<4xbf16> to vector<4xf32>
+ %out = nvgpu.extf %in : vector<4xbf16> to vector<4xf32>
return %out : vector<4xf32>
}
@@ -283,7 +283,7 @@ func.func @fpext_bf16_to_f32(%in : vector<4xbf16>) -> vector<4xf32> {
func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.fpext %arg0 : vector<4xf16> to vector<4xf64>
+ %out = nvgpu.extf %arg0 : vector<4xf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -291,7 +291,7 @@ func.func @fpext_f16_to_f64(%arg0: vector<4xf16>) -> vector<4xf64> {
func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
// CHECK: llvm.fpext %{{.*}} : vector<4xbf16> to vector<4xf64>
// CHECK-NOT: llvm.fpext
- %out = nvgpu.convert.fpext %arg0 : vector<4xbf16> to vector<4xf64>
+ %out = nvgpu.extf %arg0 : vector<4xbf16> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -299,7 +299,7 @@ func.func @fpext_bf16_to_f64(%arg0: vector<4xbf16>) -> vector<4xf64> {
func.func @fpext_f32_to_f64(%arg0: vector<4xf32>) -> vector<4xf64> {
// CHECK-NOT: nvvm
// CHECK: llvm.fpext %{{.*}} : vector<4xf32> to vector<4xf64>
- %out = nvgpu.convert.fpext %arg0 : vector<4xf32> to vector<4xf64>
+ %out = nvgpu.extf %arg0 : vector<4xf32> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -309,7 +309,7 @@ func.func @fpext_f8_to_f64(%arg0: vector<4xf8E4M3FN>) -> vector<4xf64> {
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
+ %out = nvgpu.extf %arg0 : vector<4xf8E4M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -318,7 +318,7 @@ func.func @fpext_f4_to_f64(%arg0: vector<8xf4E2M1FN>) -> vector<8xf64> {
// CHECK: nvvm.convert.f4x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.fpext %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
+ %out = nvgpu.extf %arg0 : vector<8xf4E2M1FN> to vector<8xf64>
return %out : vector<8xf64>
}
@@ -328,7 +328,7 @@ func.func @fpext_e2m3_to_f64(%arg0: vector<4xf6E2M3FN>) -> vector<4xf64> {
// CHECK: nvvm.convert.f6x2.to.f16x2
// CHECK: llvm.fpext {{.*}} to {{.*}}f64
// CHECK-NOT: llvm.fpext {{.*}} to {{.*}}f32
- %out = nvgpu.convert.fpext %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
+ %out = nvgpu.extf %arg0 : vector<4xf6E2M3FN> to vector<4xf64>
return %out : vector<4xf64>
}
@@ -342,7 +342,7 @@ func.func @fpext_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f16 from vector<1xf16>
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ %out = nvgpu.extf %in : f8E4M3FN to f16
return %out : f16
}
@@ -357,7 +357,7 @@ func.func @fpext_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
// CHECK: vector.extract {{.*}}[0] : f32 from vector<1xf32>
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ %out = nvgpu.extf %in : f8E4M3FN to f32
return %out : f32
}
@@ -369,7 +369,7 @@ func.func @fpext_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK: vector.shape_cast %[[IN]] : vector<2x4xf8E4M3FN> to vector<8xf8E4M3FN>
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf16>
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.extf %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -382,7 +382,7 @@ func.func @fpext_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.shape_cast {{.*}} to vector<2x4xf32>
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.extf %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -394,7 +394,7 @@ func.func @fpext_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK: vector.insert_strided_slice %[[IN]]
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.extf %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -407,7 +407,7 @@ func.func @fpext_v3_f8_to_f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: llvm.fpext
// CHECK: vector.extract_strided_slice
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.extf %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
@@ -421,7 +421,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK: vector.extract_strided_slice
// CHECK: vector.shape_cast {{.*}} to vector<3x1xf16>
- %out = nvgpu.convert.fpext %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
+ %out = nvgpu.extf %in : vector<3x1xf8E4M3FN> to vector<3x1xf16>
return %out : vector<3x1xf16>
}
@@ -431,7 +431,7 @@ func.func @fpext_v3x1_f8_to_f16(%in : vector<3x1xf8E4M3FN>) -> vector<3x1xf16> {
func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK: nvvm.convert.f8x2.to.f16x2
// CHECK-SAME: relu = true
- %out = nvgpu.convert.fpext %in {relu = true}
+ %out = nvgpu.extf %in {relu = true}
: vector<8xf8E4M3FN> to vector<8xf16>
return
}
@@ -447,7 +447,7 @@ func.func @fpext_f8_to_f16_relu(%in : vector<8xf8E4M3FN>) {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f16
+ %out = nvgpu.extf %in : f8E4M3FN to f16
return %out : f16
}
@@ -456,7 +456,7 @@ func.func @e2e_scalar_f8_to_f16(%in : f8E4M3FN) -> f16 {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
+ %out = nvgpu.extf %in : vector<2x4xf8E4M3FN> to vector<2x4xf16>
return %out : vector<2x4xf16>
}
@@ -466,7 +466,7 @@ func.func @e2e_v2x4_f8_to_f16(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf16> {
// CHECK-E2E: nvvm.convert.f8x2.to.f16x2
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf16>
+ %out = nvgpu.extf %in : vector<3xf8E5M2> to vector<3xf16>
return %out : vector<3xf16>
}
@@ -480,7 +480,7 @@ func.func @e2e_v3f8_to_v3f16(%in : vector<3xf8E5M2>) -> vector<3xf16> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
- %out = nvgpu.convert.fpext %in : f8E4M3FN to f32
+ %out = nvgpu.extf %in : f8E4M3FN to f32
return %out : f32
}
@@ -490,7 +490,7 @@ func.func @e2e_scalar_f8_to_f32(%in : f8E4M3FN) -> f32 {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
- %out = nvgpu.convert.fpext %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
+ %out = nvgpu.extf %in : vector<2x4xf8E4M3FN> to vector<2x4xf32>
return %out : vector<2x4xf32>
}
@@ -501,6 +501,6 @@ func.func @e2e_v2x4_f8_to_f32(%in : vector<2x4xf8E4M3FN>) -> vector<2x4xf32> {
// CHECK-E2E: llvm.fpext
// CHECK-E2E: return
func.func @e2e_v3f8_to_v3f32(%in : vector<3xf8E5M2>) -> vector<3xf32> {
- %out = nvgpu.convert.fpext %in : vector<3xf8E5M2> to vector<3xf32>
+ %out = nvgpu.extf %in : vector<3xf8E5M2> to vector<3xf32>
return %out : vector<3xf32>
}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-truncf-large.mlir
similarity index 100%
rename from mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc-large.mlir
rename to mlir/test/Conversion/NVGPUToNVVM/nvgpu-truncf-large.mlir
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-truncf.mlir
similarity index 100%
rename from mlir/test/Conversion/NVGPUToNVVM/nvgpu-convert-fptrunc.mlir
rename to mlir/test/Conversion/NVGPUToNVVM/nvgpu-truncf.mlir
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
deleted file mode 100644
index 26c3afc5da358..0000000000000
--- a/mlir/test/Dialect/NVGPU/nvgpu-convert-fpext-invalid.mlir
+++ /dev/null
@@ -1,82 +0,0 @@
-// RUN: mlir-opt -split-input-file -verify-diagnostics %s
-
-// -----
-
-func.func @fpext_wider(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op result type 'f4E2M1FN' must be wider than operand type 'f8E5M2'}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
- return
-}
-
-// -----
-
-func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op result type must be 16, 32, or 64 bitwidth, but got 8}}
- %out = nvgpu.convert.fpext %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
- return
-}
-
-// -----
-
-func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op expects bf16 or f32 output type when input type is e8m0.}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E8M0FNU> to vector<16xf16>
- return
-}
-
-// -----
-
-func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
- %out = nvgpu.convert.fpext %in {rnd = #nvvm.fp_rnd_mode<rz>}
- : vector<16xf8E5M2> to vector<16xf16>
- return
-}
-
-// -----
-
-func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op relu is not supported for bf16 destination}}
- %out = nvgpu.convert.fpext %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
- return
-}
-
-// -----
-
-func.func @fpext_shape_mismatch(%in : vector<16xf8E5M2>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output shapes must match}}
- %out = nvgpu.convert.fpext %in : vector<16xf8E5M2> to vector<8xf16>
- return
-}
-
-// -----
-
-func.func @fpext_scalar_vector_mismatch(%in : f8E4M3FN) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must both be scalars or both be vectors/tensors}}
- %out = nvgpu.convert.fpext %in : f8E4M3FN to vector<1xf16>
- return
-}
-
-// -----
-
-func.func @fpext_rank0_tensor(%in : tensor<f8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op rank-0 shaped types are not supported, use scalar type instead}}
- %out = nvgpu.convert.fpext %in : tensor<f8E4M3FN> to tensor<f16>
- return
-}
-
-// -----
-
-func.func @fpext_container_mismatch(%in : vector<4xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op input and output must be the same container type (both vector or both tensor)}}
- %out = nvgpu.convert.fpext %in : vector<4xf8E4M3FN> to tensor<4xf16>
- return
-}
-
-// -----
-
-func.func @fpext_unranked_tensor(%in : tensor<*xf8E4M3FN>) {
- // expected-error @+1 {{'nvgpu.convert.fpext' op unranked tensor types are not supported}}
- %out = nvgpu.convert.fpext %in : tensor<*xf8E4M3FN> to tensor<*xf16>
- return
-}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-extf-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-extf-invalid.mlir
new file mode 100644
index 0000000000000..874ea7ab0e6fc
--- /dev/null
+++ b/mlir/test/Dialect/NVGPU/nvgpu-extf-invalid.mlir
@@ -0,0 +1,82 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+
+// -----
+
+func.func @fpext_wider(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.extf' op result type 'f4E2M1FN' must be wider than operand type 'f8E5M2'}}
+ %out = nvgpu.extf %in : vector<16xf8E5M2> to vector<16xf4E2M1FN>
+ return
+}
+
+// -----
+
+func.func @fpext_dst_bitwidth(%in : vector<16xf4E2M1FN>) {
+ // expected-error @+1 {{'nvgpu.extf' op result type must be 16, 32, or 64 bitwidth, but got 8}}
+ %out = nvgpu.extf %in : vector<16xf4E2M1FN> to vector<16xf8E4M3FN>
+ return
+}
+
+// -----
+
+func.func @fpext_e8m0_to_f16(%in : vector<16xf8E8M0FNU>) {
+ // expected-error @+1 {{'nvgpu.extf' op expects bf16 or f32 output type when input type is e8m0.}}
+ %out = nvgpu.extf %in : vector<16xf8E8M0FNU> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_bad_rounding(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.extf' op expects RN rounding mode, but got #nvvm.fp_rnd_mode<rz>}}
+ %out = nvgpu.extf %in {rnd = #nvvm.fp_rnd_mode<rz>}
+ : vector<16xf8E5M2> to vector<16xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_relu_bf16(%in : vector<8xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.extf' op relu is not supported for bf16 destination}}
+ %out = nvgpu.extf %in {relu = true} : vector<8xf8E5M2> to vector<8xbf16>
+ return
+}
+
+// -----
+
+func.func @fpext_shape_mismatch(%in : vector<16xf8E5M2>) {
+ // expected-error @+1 {{'nvgpu.extf' op input and output shapes must match}}
+ %out = nvgpu.extf %in : vector<16xf8E5M2> to vector<8xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_scalar_vector_mismatch(%in : f8E4M3FN) {
+ // expected-error @+1 {{'nvgpu.extf' op input and output must both be scalars or both be vectors/tensors}}
+ %out = nvgpu.extf %in : f8E4M3FN to vector<1xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_rank0_tensor(%in : tensor<f8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.extf' op rank-0 shaped types are not supported, use scalar type instead}}
+ %out = nvgpu.extf %in : tensor<f8E4M3FN> to tensor<f16>
+ return
+}
+
+// -----
+
+func.func @fpext_container_mismatch(%in : vector<4xf8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.extf' op input and output must be the same container type (both vector or both tensor)}}
+ %out = nvgpu.extf %in : vector<4xf8E4M3FN> to tensor<4xf16>
+ return
+}
+
+// -----
+
+func.func @fpext_unranked_tensor(%in : tensor<*xf8E4M3FN>) {
+ // expected-error @+1 {{'nvgpu.extf' op unranked tensor types are not supported}}
+ %out = nvgpu.extf %in : tensor<*xf8E4M3FN> to tensor<*xf16>
+ return
+}
diff --git a/mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir b/mlir/test/Dialect/NVGPU/nvgpu-truncf-invalid.mlir
similarity index 100%
rename from mlir/test/Dialect/NVGPU/nvgpu-convert-fptrunc-invalid.mlir
rename to mlir/test/Dialect/NVGPU/nvgpu-truncf-invalid.mlir
>From 9da2835665afefdd93f3adc06cb50edec612e24c Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 3 Jul 2026 12:46:13 +0000
Subject: [PATCH 09/11] fix formatting
---
mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index bb0bbc2d0a284..d2f7fe768930b 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -2121,10 +2121,9 @@ static Value createExtConversion(ImplicitLocOpBuilder &b, MLIRContext *ctx,
llvm_unreachable("unhandled FPExtConvOp");
}
-static LogicalResult lowerExtf(nvgpu::ExtfOp op,
- nvgpu::ExtfOp::Adaptor adaptor,
- ConversionPatternRewriter &rewriter,
- const LLVMTypeConverter *typeConverter) {
+static LogicalResult lowerExtf(nvgpu::ExtfOp op, nvgpu::ExtfOp::Adaptor adaptor,
+ ConversionPatternRewriter &rewriter,
+ const LLVMTypeConverter *typeConverter) {
MLIRContext *ctx = op.getContext();
ImplicitLocOpBuilder b(op->getLoc(), rewriter);
IntegerType i8Ty = b.getI8Type();
@@ -2271,8 +2270,7 @@ struct NVGPUTruncfOpLowering : public ConvertOpToLLVMPattern<nvgpu::TruncfOp> {
}
};
-struct NVGPUExtfOpLowering
- : public ConvertOpToLLVMPattern<nvgpu::ExtfOp> {
+struct NVGPUExtfOpLowering : public ConvertOpToLLVMPattern<nvgpu::ExtfOp> {
using ConvertOpToLLVMPattern<nvgpu::ExtfOp>::ConvertOpToLLVMPattern;
LogicalResult
@@ -2434,6 +2432,6 @@ void mlir::populateNVGPUToNVVMConversionPatterns(
NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
NVGPUMmaSparseSyncLowering, NVGPURcpOpLowering>(converter);
- patterns.add<NVGPUTruncfCanonicalizePattern,
- NVGPUExtfCanonicalizePattern>(patterns.getContext());
+ patterns.add<NVGPUTruncfCanonicalizePattern, NVGPUExtfCanonicalizePattern>(
+ patterns.getContext());
}
>From 901c37448621ec25ebc7d5204a0bfa23388b38f3 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 3 Jul 2026 12:56:56 +0000
Subject: [PATCH 10/11] convert single use function to lambda
---
mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index d2f7fe768930b..67d3b2e86293a 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -1751,11 +1751,6 @@ static std::optional<FPKind> classifyFPType(Type t) {
return std::nullopt;
}
-/// Number of source-side i32 register slots consumed by each NVVM convert Op.
-static int getNumSrcI32PerConvert(FPKind src) {
- return src == FPKind::F32 ? 2 : 1;
-}
-
/// Conversion op identifier for nvgpu.truncf lowering dispatch table.
enum class FPTruncConvOp {
F32x2_TO_F16x2,
@@ -1985,6 +1980,11 @@ static LogicalResult lowerTruncf(nvgpu::TruncfOp op,
return rewriter.notifyMatchFailure(
op, "unsupported type combination for truncation");
FPTruncConvOp convOp = convEntry->convOp;
+
+ // Number of source-side i32 register slots consumed by each NVVM convert Op.
+ auto getNumSrcI32PerConvert = [](FPKind src) {
+ return src == FPKind::F32 ? 2 : 1;
+ };
int numSrcI32PerConv = getNumSrcI32PerConvert(convEntry->src);
// STEP 3: pack conversion results into destination i32 vector.
>From 547ebf0ea052d560f2d1d2d3a4f37855be2c79da Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Thu, 9 Jul 2026 11:26:53 +0000
Subject: [PATCH 11/11] address comment
---
mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 8dd67411292dd..817cb8595f9ca 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -764,7 +764,9 @@ LogicalResult TruncfOp::verify() {
return emitOpError("expects RZ or RP rounding mode when result type is "
"e8m0, but got ")
<< getRndAttr();
- } else if (rnd == mlir::NVVM::FPRoundingMode::RS) {
+ }
+
+ if (rnd == mlir::NVVM::FPRoundingMode::RS) {
// TODO: Currently, we only support conversions which fit into a single i32
// register. Support f32->f8/f6/f4 conversions with RS rounding.
if (!(srcBitWidth == 32 && dstBitWidth == 16))
@@ -773,19 +775,24 @@ LogicalResult TruncfOp::verify() {
<< srcType << " -> " << dstType;
if (!getRandomBits())
return emitOpError("random_bits operand is required with RS rounding");
- } else if (srcType.isF64() && dstBitWidth >= 16) {
+ }
+
+ if (srcType.isF64() && dstBitWidth >= 16) {
if (rnd != mlir::NVVM::FPRoundingMode::RN)
return emitOpError("expects RN rounding mode for f64 input, but got ")
<< getRndAttr();
- } else if (srcBitWidth == 32 && dstBitWidth == 16) {
+ }
+
+ if (srcBitWidth == 32 && dstBitWidth == 16) {
if (rnd != mlir::NVVM::FPRoundingMode::RN &&
rnd != mlir::NVVM::FPRoundingMode::RZ)
return emitOpError("expects RN or RZ rounding mode for f32 to f16/bf16, "
"but got ")
<< getRndAttr();
- } else if (rnd != mlir::NVVM::FPRoundingMode::RN) {
- return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
}
+
+ if (rnd != mlir::NVVM::FPRoundingMode::RN)
+ return emitOpError("expects RN rounding mode, but got ") << getRndAttr();
if (getRandomBits() && rnd != mlir::NVVM::FPRoundingMode::RS)
return emitOpError("random_bits can only be used with RS rounding mode");
More information about the Mlir-commits
mailing list