[Mlir-commits] [mlir] [MLIR][XeVM] Support all SPIR-V vector lengths in truncf and extf (PR #217768)
Sang Ik Lee
llvmlistbot at llvm.org
Wed Aug 26 13:04:29 PDT 2026
https://github.com/silee2 updated https://github.com/llvm/llvm-project/pull/217768
>From 752ba0f86406ad009b74bf6e4d18ffc7e8318b03 Mon Sep 17 00:00:00 2001
From: "Lee, Sang Ik" <sang.ik.lee at intel.com>
Date: Thu, 20 Aug 2026 21:30:52 +0000
Subject: [PATCH 1/2] [MLIR][XeVM] Support all SPIR-V vector lengths in truncf
and extf
The fp8 and fp4 conversions only lowered a 16 element vector. Lower every
SPIR-V vector length, 2, 3, 4, 8 and 16, for both formats and for f16 and bf16
operands, following IGC's SPV_INTEL_fp_conversions implementation.
Two fp4 values pack into a single byte, and SPIR-V uses a scalar where it has
no one element vector, so the packed side of that conversion is a scalar. The
verifier previously required both operands to be vectors or both scalars, which
rejected it. It now checks that the packed side is exactly wide enough to hold
the values it carries, which both permits that case and catches mismatched
lengths that used to pass.
Also fixes two latent bugs in the shuffle legalization that the new IR reaches:
a null dereference when a shuffle reads a bitcast of a scalar, and a mask
computed by a division that could drop a remainder, which built an invalid
shuffle when the extracted slice did not cover whole source elements.
Adds conversion tests for each length, and end to end round trip tests for both
formats with f16 and bf16 operands.
---
mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp | 361 +++++++++++-------
mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp | 54 ++-
.../xevm_fp_conversion_lengths.mlir | 219 +++++++++++
.../XeVMToLLVM/xevm_mx-to-llvm.mlir | 36 +-
mlir/test/Dialect/LLVMIR/invalid.mlir | 8 +-
...vm_truncf_extf_roundtrip_bf16_lengths.mlir | 192 ++++++++++
...runcf_extf_roundtrip_fp4_bf16_lengths.mlir | 192 ++++++++++
...evm_truncf_extf_roundtrip_fp4_lengths.mlir | 188 +++++++++
...evm_truncf_extf_roundtrip_fp8_lengths.mlir | 188 +++++++++
9 files changed, 1277 insertions(+), 161 deletions(-)
create mode 100644 mlir/test/Conversion/XeVMToLLVM/xevm_fp_conversion_lengths.mlir
create mode 100644 mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
create mode 100644 mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
create mode 100644 mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
create mode 100644 mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
diff --git a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
index c51da4d5d4d3d..a6c7de1dd98e0 100644
--- a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
+++ b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
@@ -16,7 +16,9 @@
#include "mlir/Pass/Pass.h"
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/MathExtras.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Matchers.h"
@@ -1132,6 +1134,33 @@ class SubgroupOpWorkitemOpToOCLPattern : public OpConversionPattern<OpType> {
}
};
+/// SPIR-V, and so the OpenCL builtins the float conversions call into, only
+/// provides vector types of 2, 3, 4, 8 and 16 elements.
+static bool isSupportedSPIRVVectorLength(int64_t numElements) {
+ return llvm::is_contained({2, 3, 4, 8, 16}, numElements);
+}
+
+/// Bitcasts `val` to `ty` unless it already has that type.
+static Value castIfNeeded(ConversionPatternRewriter &rewriter, Location loc,
+ Type ty, Value val) {
+ if (val.getType() == ty)
+ return val;
+ return LLVM::BitcastOp::create(rewriter, loc, ty, val);
+}
+
+/// Selects `numElements` leading elements of the vector `val`. Used to drop the
+/// padding the 3 element case needs, as the hardware conversions work on whole
+/// pairs of elements.
+static Value takeLeadingElements(ConversionPatternRewriter &rewriter,
+ Location loc, Value val, int64_t numElements) {
+ auto vecTy = cast<VectorType>(val.getType());
+ if (vecTy.getNumElements() == numElements)
+ return val;
+ SmallVector<int32_t> mask =
+ llvm::to_vector(llvm::seq<int32_t>(0, static_cast<int32_t>(numElements)));
+ return LLVM::ShuffleVectorOp::create(rewriter, loc, val, val, mask);
+}
+
class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult
@@ -1140,34 +1169,23 @@ class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
// Supported source and result types are resticted for now.
auto srcEtype = op.getSrcEtype().getEtype();
auto dstEtype = op.getDstEtype().getEtype();
- // Currently only 16 input elements are supported as
- // - Any vector beyond 16 elements not a valid OpenCL vector.
- // - 2D block load can only load up to 16 16bit elements per lane.
- // Widest load is 8x16xi32 with 16 lanes, which is 16 16bit
- // elements per lane.
- // - mma_mx A and B operands need more than 16 elements per lane
+ // The conversions are provided as OpenCL builtins, one per vector length,
+ // so only the SPIR-V vector lengths can be lowered. A wider conversion has
+ // to be split into several ops before reaching this pattern.
//
- // Conversion is done in batches depending on the dst type.
- // batch_size =
- // 16 if dst type == fp8
- // 8 if dst type == fp4
- // For num_elem > batch_size
- // convert batch of batch_size
- // cast batch to i32 elem type vector
- // concat batches by shufflevector
- // For num_elem = batch_size
- // use API for conversion
// Scalar case is not supported until usage case become clear.
auto vecSrcTy = dyn_cast<VectorType>(op.getSrc().getType());
if (!vecSrcTy) {
return rewriter.notifyMatchFailure(op, "Scalar src is not supported.");
}
- if (vecSrcTy.getNumElements() != 16)
+ int64_t numElements = vecSrcTy.getNumElements();
+ if (!isSupportedSPIRVVectorLength(numElements))
return rewriter.notifyMatchFailure(
- op, "Only vector src of 16 elements is supported");
- auto vecDstTy = dyn_cast<VectorType>(op.getDst().getType());
- if (!vecDstTy)
- return rewriter.notifyMatchFailure(op, "Scalar dst is not supported.");
+ op, "src vector length must be 2, 3, 4, 8 or 16");
+ // The destination is scalar only where the packed values fit in one byte,
+ // which SPIR-V spells as a scalar rather than a one element vector.
+ Type dstTy = op.getDst().getType();
+ Location loc = op.getLoc();
Value src = op.getSrc();
auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
/*other=*/LLVM::ModRefInfo::NoModRef,
@@ -1181,84 +1199,130 @@ class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
// Handle the case where dst type is fp4 first.
if (dstEtype == TruncfDstElemTypes::E2M1) {
- // Convert 8 elements at a time.
- // To convert 8 elements, vector<8xf16>:
- // Use:
- // uint __builtin_IB_dnscl_hf16(uint, uint, 1, 0)
- // uint __builtin_IB_dnscl_hf16(uint, uint, 1, 3)
- // llvm.or
- Value cast = LLVM::BitcastOp::create(
- rewriter, op.getLoc(), VectorType::get(8, rewriter.getI32Type()),
- src);
+ // `__builtin_IB_dnscl_{hf16,bf16}(uint a, uint b, convert_to, mode)`
+ // takes two dwords, each holding two source elements, and packs each pair
+ // into one byte of the result dword. `mode` picks which bytes of that
+ // dword are written: mode 0 writes bytes 0 and 2, mode 2 writes bytes 1
+ // and 3. Two calls with complementary modes therefore OR together into
+ // one fully packed dword covering eight source elements.
+ //
+ // A pair of elements is the conversion granularity, so an odd length is
+ // padded up and the spare nibble left undefined.
+ constexpr int kDnsclConvertToE2M1 = 1;
+ constexpr int kDnsclModeBytes02 = 0;
+ constexpr int kDnsclModeBytes13 = 2;
+ // One dword lane per element pair, and one result byte per lane. The op
+ // verifier has already checked that the destination is exactly that wide.
+ int64_t numLanes = llvm::divideCeil(numElements, 2);
+
+ Type i32Ty = rewriter.getI32Type();
+ Type i8Ty = rewriter.getI8Type();
+ // Pad an odd length up to a whole number of pairs, then view the source
+ // as dword lanes.
+ Value padded = src;
+ if (numElements != numLanes * 2) {
+ SmallVector<int32_t> mask = llvm::to_vector(
+ llvm::seq<int32_t>(0, static_cast<int32_t>(numElements)));
+ // The padding element is never read back, so any valid index will do.
+ mask.append(static_cast<size_t>(numLanes * 2 - numElements), 0);
+ padded = LLVM::ShuffleVectorOp::create(rewriter, loc, src, src, mask);
+ }
+ // A single lane is passed as a bare i32 rather than a one element vector,
+ // which SPIR-V has no type for.
+ Value laneVec;
+ if (numLanes > 1)
+ laneVec = LLVM::BitcastOp::create(
+ rewriter, loc, VectorType::get(numLanes, i32Ty), padded);
+ else
+ laneVec = LLVM::BitcastOp::create(rewriter, loc, i32Ty, padded);
+ auto getLane = [&](int64_t idx) -> Value {
+ if (numLanes == 1)
+ return laneVec;
+ Value pos =
+ LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), idx);
+ return LLVM::ExtractElementOp::create(rewriter, loc, laneVec, pos)
+ ->getResult(0);
+ };
std::string fnName = "__builtin_IB_dnscl_";
fnName += (srcEtype == TruncfSrcElemTypes::F16) ? "hf16" : "bf16";
- auto genDnscl = [&](Value input, Value idx0, Value idx1, Value dstTy,
- Value mode) -> Value {
- Value arg1 =
- LLVM::ExtractElementOp::create(rewriter, op.getLoc(), input, idx0)
- ->getResult(0);
- Value arg2 =
- LLVM::ExtractElementOp::create(rewriter, op.getLoc(), input, idx1)
- ->getResult(0);
- SmallVector<Type> argTypes{arg1.getType(), arg2.getType(),
- dstTy.getType(), mode.getType()};
- SmallVector<Value> args{arg1, arg2, dstTy, mode};
- Value dnscl = createDeviceFunctionCall(
- rewriter, fnName, rewriter.getI32Type(), argTypes,
- args, {}, funcAttrs, op.getOperation())
- ->getResult(0);
- return dnscl;
+ Value convertTo =
+ LLVM::ConstantOp::create(rewriter, loc, i32Ty, kDnsclConvertToE2M1);
+ auto genDnscl = [&](Value lo, Value hi, int mode) -> Value {
+ Value modeVal = LLVM::ConstantOp::create(rewriter, loc, i32Ty, mode);
+ SmallVector<Type> argTypes{lo.getType(), hi.getType(),
+ convertTo.getType(), modeVal.getType()};
+ SmallVector<Value> args{lo, hi, convertTo, modeVal};
+ return createDeviceFunctionCall(rewriter, fnName, i32Ty, argTypes, args,
+ {}, funcAttrs, op.getOperation())
+ ->getResult(0);
};
- Value zero = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 0);
- Value one = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 1);
- Value two = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 2);
- Value three = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 3);
- Value even = genDnscl(cast, zero, two, one, zero);
- Value odd = genDnscl(cast, one, three, one, two);
- Value firstHalf = LLVM::OrOp::create(rewriter, op.getLoc(), even, odd);
- Value four = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 4);
- Value five = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 5);
- Value six = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 6);
- Value seven = LLVM::ConstantOp::create(rewriter, op.getLoc(),
- rewriter.getI32Type(), 7);
- even = genDnscl(cast, four, six, one, zero);
- odd = genDnscl(cast, five, seven, one, two);
- Value secondHalf = LLVM::OrOp::create(rewriter, op.getLoc(), even, odd);
- // Create vector<2xi32> from two i32 values and then bitcast to
- // vector<8xi8> to match the dst type.
- Value combined = LLVM::UndefOp::create(
- rewriter, op.getLoc(), VectorType::get(2, rewriter.getI32Type()));
- combined = LLVM::InsertElementOp::create(rewriter, op.getLoc(), combined,
- firstHalf, zero)
- ->getResult(0);
- combined = LLVM::InsertElementOp::create(rewriter, op.getLoc(), combined,
- secondHalf, one)
- ->getResult(0);
- Value result =
- LLVM::BitcastOp::create(rewriter, op.getLoc(), vecDstTy, combined);
- rewriter.replaceOp(op, result);
+ Value result;
+ if (numLanes <= 2) {
+ // Fewer than four lanes cannot fill a dword, so a single call is made
+ // and the written bytes, 0 and 2, are compacted afterwards.
+ Value lo = getLane(0);
+ Value hi =
+ numLanes == 2
+ ? getLane(1)
+ : LLVM::UndefOp::create(rewriter, loc, i32Ty)->getResult(0);
+ Value dword = genDnscl(lo, hi, kDnsclModeBytes02);
+ if (numLanes == 1) {
+ // A single byte, so the low one, is all that is kept.
+ result = LLVM::TruncOp::create(rewriter, loc, i8Ty, dword);
+ } else {
+ Value bytes = LLVM::BitcastOp::create(
+ rewriter, loc, VectorType::get(4, i8Ty), dword);
+ result = LLVM::ShuffleVectorOp::create(rewriter, loc, bytes, bytes,
+ ArrayRef<int32_t>{0, 2});
+ }
+ } else {
+ // Four lanes, eight source elements, per fully packed dword.
+ SmallVector<Value> dwords;
+ for (int64_t base = 0; base < numLanes; base += 4) {
+ // Each lane is bound to a name first: `getLane` builds ops, and the
+ // order of evaluation within an argument list is unspecified, which
+ // would otherwise leave the order of the emitted ops up to the host
+ // compiler.
+ Value lane0 = getLane(base);
+ Value lane2 = getLane(base + 2);
+ Value even = genDnscl(lane0, lane2, kDnsclModeBytes02);
+ Value lane1 = getLane(base + 1);
+ Value lane3 = getLane(base + 3);
+ Value odd = genDnscl(lane1, lane3, kDnsclModeBytes13);
+ dwords.push_back(LLVM::OrOp::create(rewriter, loc, even, odd));
+ }
+ if (dwords.size() == 1) {
+ result = dwords.front();
+ } else {
+ Type packedTy = VectorType::get(dwords.size(), i32Ty);
+ result = LLVM::UndefOp::create(rewriter, loc, packedTy);
+ for (auto [idx, dword] : llvm::enumerate(dwords)) {
+ Value pos = LLVM::ConstantOp::create(rewriter, loc, i32Ty, idx);
+ result =
+ LLVM::InsertElementOp::create(rewriter, loc, result, dword, pos)
+ ->getResult(0);
+ }
+ }
+ }
+ rewriter.replaceOp(op, castIfNeeded(rewriter, loc, dstTy, result));
return success();
}
// Handle the case where dst type is fp8.
+ // The fp8 conversions come as one builtin per vector length, so the length
+ // is simply appended to the builtin name.
+ std::string lenSuffix = std::to_string(numElements);
// BF16 type needs some preprocessing before conversion,
// First extended to F32 and then truncated to F16.
if (srcEtype == TruncfSrcElemTypes::BF16) {
// Step 1: Extend to F32
- // Use float16 __builtin_IB_bftof_16(short16)
+ // Use floatN __builtin_IB_bftof_N(shortN)
src = LLVM::BitcastOp::create(
rewriter, op.getLoc(),
VectorType::get(vecSrcTy.getShape(), rewriter.getI16Type()), src);
- std::string fnName = "__builtin_IB_bftof_16";
+ std::string fnName = "__builtin_IB_bftof_" + lenSuffix;
SmallVector<Type> argTypes{src.getType()};
SmallVector<Value> args{src};
Type resTy = VectorType::get(vecSrcTy.getShape(), rewriter.getF32Type());
@@ -1266,8 +1330,8 @@ class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
{}, funcAttrs, op.getOperation())
->getResult(0);
// Step 2: Truncf to F16
- // Use half16 convert_half16(float16)
- std::string truncFnName = "convert_half16";
+ // Use halfN convert_halfN(floatN)
+ std::string truncFnName = "convert_half" + lenSuffix;
SmallVector<Type> truncArgTypes{src.getType()};
SmallVector<Value> truncArgs{src};
truncFnName = mangle(truncFnName, truncArgTypes);
@@ -1278,24 +1342,24 @@ class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
->getResult(0);
}
if (dstEtype == TruncfDstElemTypes::BF8) { // Float8E5M2Type
- // Use char16 __builtin_IB_hftobf8_16(half16)
- std::string fnName = "__builtin_IB_hftobf8_16";
+ // Use charN __builtin_IB_hftobf8_N(halfN)
+ std::string fnName = "__builtin_IB_hftobf8_" + lenSuffix;
SmallVector<Type> argTypes{src.getType()};
SmallVector<Value> args{src};
Value result =
- createDeviceFunctionCall(rewriter, fnName, vecDstTy, argTypes, args,
- {}, funcAttrs, op.getOperation())
+ createDeviceFunctionCall(rewriter, fnName, dstTy, argTypes, args, {},
+ funcAttrs, op.getOperation())
->getResult(0);
rewriter.replaceOp(op, result);
} else if (dstEtype == TruncfDstElemTypes::F8) { // Float8E4M3FNType
- // Use char16 __builtin_IB_hftohf8_16(half16)
- std::string fnName = "__builtin_IB_hftohf8_16";
+ // Use charN __builtin_IB_hftohf8_N(halfN)
+ std::string fnName = "__builtin_IB_hftohf8_" + lenSuffix;
SmallVector<Type> argTypes{src.getType()};
SmallVector<Value> args{src};
Value result =
- createDeviceFunctionCall(rewriter, fnName, vecDstTy, argTypes, args,
- {}, funcAttrs, op.getOperation())
+ createDeviceFunctionCall(rewriter, fnName, dstTy, argTypes, args, {},
+ funcAttrs, op.getOperation())
->getResult(0);
rewriter.replaceOp(op, result);
@@ -1316,13 +1380,19 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
// types are restricted for now, mirroring the truncf lowering.
auto srcEtype = op.getSrcEtype().getEtype();
auto dstEtype = op.getDstEtype().getEtype();
- // Scalar case is not supported until usage case become clear.
- auto vecSrcTy = dyn_cast<VectorType>(op.getSrc().getType());
- if (!vecSrcTy)
- return rewriter.notifyMatchFailure(op, "Scalar src is not supported.");
+ // The source is scalar only where the packed values fit in one byte, which
+ // SPIR-V spells as a scalar rather than a one element vector.
+ Type srcTy = op.getSrc().getType();
+ // Scalar dst is not supported until usage case become clear.
auto vecDstTy = dyn_cast<VectorType>(op.getDst().getType());
if (!vecDstTy)
return rewriter.notifyMatchFailure(op, "Scalar dst is not supported.");
+ // As for truncf, one builtin exists per SPIR-V vector length.
+ int64_t numElements = vecDstTy.getNumElements();
+ if (!isSupportedSPIRVVectorLength(numElements))
+ return rewriter.notifyMatchFailure(
+ op, "dst vector length must be 2, 3, 4, 8 or 16");
+ Location loc = op.getLoc();
Value src = op.getSrc();
auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
/*other=*/LLVM::ModRefInfo::NoModRef,
@@ -1336,22 +1406,24 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
// Handle the case where src type is fp4 (e2m1) first.
if (srcEtype == ExtfSrcElemTypes::E2M1) {
- // 16 fp4 values are packed into vector<8xi8>, the result is a
- // vector<16xf16> or vector<16xbf16>.
- // Use:
+ // Two fp4 values are packed per source byte, and one builtin exists per
+ // source byte count:
// uint16 __builtin_IB_shfl_idx4_lut(int lut_index)
- // uint8 __builtin_IB_shfl_idx4_to_fp16_8_packed(uint16 lut,
- // char8 source)
+ // uint __builtin_IB_shfl_idx4_to_fp16_packed(uint16 lut, char src)
+ // uintN __builtin_IB_shfl_idx4_to_fp16_N_packed(uint16 lut, charN src)
+ // Each returns one dword, holding two f16/bf16 values, per source byte.
// The lookup table selects the target format:
// 7 = e2m1 -> f16, 5 = e2m1 -> bf16.
- if (vecSrcTy.getNumElements() != 8 || vecDstTy.getNumElements() != 16)
- return rewriter.notifyMatchFailure(
- op, "fp4 src expects a vector<8xi8> src and a 16 element dst");
+ //
+ // A byte is the conversion granularity, so an odd length reads one spare
+ // value that is dropped afterwards. The op verifier has already checked
+ // that the source is exactly as wide as those bytes.
+ int64_t numBytes = llvm::divideCeil(numElements, 2);
constexpr int kLutE2M1ToF16 = 7;
constexpr int kLutE2M1ToBF16 = 5;
int lutIndex =
(dstEtype == ExtfDstElemTypes::F16) ? kLutE2M1ToF16 : kLutE2M1ToBF16;
- Value lutIdx = LLVM::ConstantOp::create(rewriter, op.getLoc(),
+ Value lutIdx = LLVM::ConstantOp::create(rewriter, loc,
rewriter.getI32Type(), lutIndex);
Type lutTy = VectorType::get(16, rewriter.getI32Type());
Value lut =
@@ -1359,33 +1431,51 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
lutTy, {lutIdx.getType()}, {lutIdx}, {},
funcAttrs, op.getOperation())
->getResult(0);
- Type packedResTy = VectorType::get(8, rewriter.getI32Type());
- SmallVector<Type> convArgTypes{lut.getType(), src.getType()};
- SmallVector<Value> convArgs{lut, src};
+ // A single byte is passed as a bare i8, and one dword returned as a bare
+ // i32, rather than as one element vectors SPIR-V has no type for.
+ Type i8Ty = rewriter.getI8Type();
+ Type i32Ty = rewriter.getI32Type();
+ std::string fnName = "__builtin_IB_shfl_idx4_to_fp16_";
+ Type argTy, packedResTy;
+ if (numBytes == 1) {
+ argTy = i8Ty;
+ packedResTy = i32Ty;
+ } else {
+ fnName += std::to_string(numBytes) + "_";
+ argTy = VectorType::get(numBytes, i8Ty);
+ packedResTy = VectorType::get(numBytes, i32Ty);
+ }
+ fnName += "packed";
+ SmallVector<Type> convArgTypes{lut.getType(), argTy};
+ SmallVector<Value> convArgs{lut, castIfNeeded(rewriter, loc, argTy, src)};
Value result =
- createDeviceFunctionCall(
- rewriter, "__builtin_IB_shfl_idx4_to_fp16_8_packed", packedResTy,
- convArgTypes, convArgs, {}, funcAttrs, op.getOperation())
+ createDeviceFunctionCall(rewriter, fnName, packedResTy, convArgTypes,
+ convArgs, {}, funcAttrs, op.getOperation())
->getResult(0);
// The builtin returns the f16/bf16 bits packed as i32, bitcast to the
- // f16/bf16 dst type.
- result = LLVM::BitcastOp::create(rewriter, op.getLoc(), vecDstTy, result);
+ // f16/bf16 dst type and drop the padding an odd length produced.
+ Type wideTy = VectorType::get(numBytes * 2, vecDstTy.getElementType());
+ result = LLVM::BitcastOp::create(rewriter, loc, wideTy, result);
+ result = takeLeadingElements(rewriter, loc, result, numElements);
rewriter.replaceOp(op, result);
return success();
}
- // Handle the case where src type is fp8 (bf8/hf8).
- // Only 16 input elements are supported, see TruncfToOCLPattern for details.
- if (vecSrcTy.getNumElements() != 16)
+ // Handle the case where src type is fp8 (bf8/hf8). One fp8 value per source
+ // byte, so source and destination lengths match.
+ auto vecSrcTy = dyn_cast<VectorType>(srcTy);
+ if (!vecSrcTy || vecSrcTy.getNumElements() != numElements)
return rewriter.notifyMatchFailure(
- op, "Only vector src of 16 elements is supported");
+ op, "fp8 src and dst must have the same number of elements");
+ std::string lenSuffix = std::to_string(numElements);
// Step 1: Extend fp8 (bf8/hf8) to F16.
- // bf8 -> half: half16 __builtin_IB_bf8tohf_16(char16)
- // hf8 -> half: half16 __builtin_IB_hf8tohf_16(char16)
+ // bf8 -> half: halfN __builtin_IB_bf8tohf_N(charN)
+ // hf8 -> half: halfN __builtin_IB_hf8tohf_N(charN)
std::string fnName = (srcEtype == ExtfSrcElemTypes::BF8)
- ? "__builtin_IB_bf8tohf_16"
- : "__builtin_IB_hf8tohf_16";
+ ? "__builtin_IB_bf8tohf_"
+ : "__builtin_IB_hf8tohf_";
+ fnName += lenSuffix;
Type f16Ty = VectorType::get(vecSrcTy.getShape(), rewriter.getF16Type());
SmallVector<Type> argTypes{src.getType()};
SmallVector<Value> args{src};
@@ -1403,8 +1493,8 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
// BF16 destination needs some postprocessing.
// First extend F16 to F32 and then truncate to BF16.
// Step 2: Extend to F32.
- // Use float16 convert_float16(half16)
- std::string convFnName = "convert_float16";
+ // Use floatN convert_floatN(halfN)
+ std::string convFnName = "convert_float" + lenSuffix;
SmallVector<Type> convArgTypes{result.getType()};
SmallVector<Value> convArgs{result};
convFnName = mangle(convFnName, convArgTypes);
@@ -1414,8 +1504,8 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
convArgs, {}, funcAttrs, op.getOperation())
->getResult(0);
// Step 3: Truncate F32 to BF16.
- // Use short16 __builtin_IB_ftobf_16(float16)
- constexpr StringRef ftobfFnName = "__builtin_IB_ftobf_16";
+ // Use shortN __builtin_IB_ftobf_N(floatN)
+ std::string ftobfFnName = "__builtin_IB_ftobf_" + lenSuffix;
SmallVector<Type> ftobfArgTypes{result.getType()};
SmallVector<Value> ftobfArgs{result};
Type i16Ty = VectorType::get(vecSrcTy.getShape(), rewriter.getI16Type());
@@ -1628,6 +1718,8 @@ class HandleVectorExtractPattern
Value srcInput = srcOp->getOperand(0);
// Create new shuffle vector op with unary input as source.
auto srcVecTy = dyn_cast<VectorType>(srcInput.getType());
+ if (!srcVecTy)
+ return failure();
auto newShuffleVecTy =
VectorType::get(mask.size(), srcVecTy.getElementType());
auto newShuffle = LLVM::ShuffleVectorOp::create(
@@ -1642,10 +1734,13 @@ class HandleVectorExtractPattern
rewriter.replaceOp(op, newUnaryOp);
} else if (isa<LLVM::BitcastOp>(srcOp)) {
Value srcInput = srcOp->getOperand(0);
- // Create new shuffle vector op with unary input as source.
+ // Create new shuffle vector op with unary input as source. A bitcast
+ // from a scalar has no slice to rewrite in terms of.
auto srcInputVecTy = dyn_cast<VectorType>(srcInput.getType());
- auto srcInputSize = srcInputVecTy.getNumElements();
auto srcResVecTy = dyn_cast<VectorType>(srcOp->getResult(0).getType());
+ if (!srcInputVecTy || !srcResVecTy)
+ return failure();
+ auto srcInputSize = srcInputVecTy.getNumElements();
auto srcResSize = srcResVecTy.getNumElements();
auto maskSize = static_cast<int32_t>(mask.size());
if (srcInputSize > srcResSize) {
@@ -1656,7 +1751,9 @@ class HandleVectorExtractPattern
}
auto maskScale = srcResSize / srcInputSize;
if (maskScale != 1) {
- if (mask[0] % maskScale != 0) {
+ // The slice has to start at, and cover, whole source elements to be
+ // expressible in terms of the bitcast source.
+ if (mask[0] % maskScale != 0 || maskSize % maskScale != 0) {
return failure();
}
// Create a new mask that maps to the source vector
@@ -1668,8 +1765,8 @@ class HandleVectorExtractPattern
}
mask = newMask;
}
- auto newShuffleVecTy =
- VectorType::get(srcInputSize, srcInputVecTy.getElementType());
+ auto newShuffleVecTy = VectorType::get(
+ static_cast<int64_t>(mask.size()), srcInputVecTy.getElementType());
auto newShuffle = LLVM::ShuffleVectorOp::create(
rewriter, loc, newShuffleVecTy, srcInput, srcInput, mask);
// Create new unary op with new shuffle as input.
@@ -1701,6 +1798,8 @@ class HandleVectorExtractPattern
if (loadAddrSpace != 0)
return failure();
auto loadTy = dyn_cast<VectorType>(loadOp.getType());
+ if (!loadTy)
+ return failure();
auto elemTy = loadTy.getElementType();
auto firstIndex = mask[0];
auto newVecTy = VectorType::get(mask.size(), elemTy);
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
index e8b3c7065f880..df4946d34e646 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -358,30 +358,68 @@ LogicalResult MMAMxOp::verify() {
return success();
}
+/// Number of bits one narrow float value occupies. The narrow values of a
+/// `xevm.truncf` destination, or a `xevm.extf` source, are packed into whole
+/// bytes, so a sub-byte format fits several values per byte.
+static int64_t getNarrowFloatBitWidth(TruncfDstElemTypes etype) {
+ return etype == TruncfDstElemTypes::E2M1 ? 4 : 8;
+}
+static int64_t getNarrowFloatBitWidth(ExtfSrcElemTypes etype) {
+ return etype == ExtfSrcElemTypes::E2M1 ? 4 : 8;
+}
+
+/// Number of values `ty` holds: its length if it is a vector, and one
+/// otherwise. SPIR-V has no vector of length one and uses a scalar instead, so
+/// a conversion of two fp4 values, which pack into a single byte, has a scalar
+/// on its packed side.
+static int64_t getNumValues(Type ty) {
+ if (auto vecTy = dyn_cast<VectorType>(ty))
+ return vecTy.getNumElements();
+ return 1;
+}
+
+/// Total bit width of `ty`, which is a scalar or a vector of a scalar.
+static int64_t getPackedBitWidth(Type ty) {
+ if (auto vecTy = dyn_cast<VectorType>(ty))
+ return vecTy.getNumElements() * vecTy.getElementTypeBitWidth();
+ return ty.getIntOrFloatBitWidth();
+}
+
+/// Verifies that `packedTy` is exactly wide enough to hold `numValues` values
+/// of `narrowBits` bits each, rounded up to whole bytes.
+static LogicalResult verifyPackedWidth(Operation *op, StringRef packedName,
+ Type packedTy, int64_t numValues,
+ int64_t narrowBits) {
+ int64_t expected = llvm::alignTo(numValues * narrowBits, 8);
+ int64_t actual = getPackedBitWidth(packedTy);
+ if (actual != expected)
+ return op->emitOpError()
+ << packedName << " should be " << expected << " bits wide to hold "
+ << numValues << " value(s) of " << narrowBits << " bits, but it is "
+ << actual;
+ return success();
+}
+
LogicalResult TruncfOp::verify() {
Type srcTy = getSrc().getType();
Type dstTy = getDst().getType();
- if (isa<VectorType>(srcTy) != isa<VectorType>(dstTy))
- return emitOpError("both src and dst should be vector types or both should "
- "be scalar types");
if (getElementTypeOrSelf(srcTy).getIntOrFloatBitWidth() <=
getElementTypeOrSelf(dstTy).getIntOrFloatBitWidth())
return emitError(
"dst element bitwidth should be less than src element bitwidth");
- return success();
+ return verifyPackedWidth(*this, "dst", dstTy, getNumValues(srcTy),
+ getNarrowFloatBitWidth(getDstEtype().getEtype()));
}
LogicalResult ExtfOp::verify() {
Type srcTy = getSrc().getType();
Type dstTy = getDst().getType();
- if (isa<VectorType>(srcTy) != isa<VectorType>(dstTy))
- return emitOpError("both src and dst should be vector types or both should "
- "be scalar types");
if (getElementTypeOrSelf(srcTy).getIntOrFloatBitWidth() >=
getElementTypeOrSelf(dstTy).getIntOrFloatBitWidth())
return emitError(
"dst element bitwidth should be greater than src element bitwidth");
- return success();
+ return verifyPackedWidth(*this, "src", srcTy, getNumValues(dstTy),
+ getNarrowFloatBitWidth(getSrcEtype().getEtype()));
}
LogicalResult
diff --git a/mlir/test/Conversion/XeVMToLLVM/xevm_fp_conversion_lengths.mlir b/mlir/test/Conversion/XeVMToLLVM/xevm_fp_conversion_lengths.mlir
new file mode 100644
index 0000000000000..fa2183f358798
--- /dev/null
+++ b/mlir/test/Conversion/XeVMToLLVM/xevm_fp_conversion_lengths.mlir
@@ -0,0 +1,219 @@
+// RUN: mlir-opt --convert-xevm-to-llvm --split-input-file %s | FileCheck %s
+
+// The fp8 and fp4 conversions are provided as one builtin per vector length.
+// This file covers the SPIR-V vector lengths other than 16, which
+// xevm_mx-to-llvm.mlir covers.
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_hftobf8_2(vector<2xf16>) -> vector<2xi8>
+// CHECK-LABEL: llvm.func @truncf_f16_to_bf8_v2
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>
+llvm.func @truncf_f16_to_bf8_v2(%src: vector<2xf16>) -> vector<2xi8> {
+ // CHECK: %[[RES:.*]] = llvm.call spir_funccc @__builtin_IB_hftobf8_2(%[[ARG0]])
+ // CHECK-SAME: : (vector<2xf16>) -> vector<2xi8>
+ %dst = xevm.truncf %src { src_etype = f16, dst_etype = bf8 } : (vector<2xf16>) -> vector<2xi8>
+ llvm.return %dst : vector<2xi8>
+}
+
+// -----
+
+// A 3 element vector is a SPIR-V vector length, and the builtins provide it.
+// CHECK: llvm.func spir_funccc @__builtin_IB_hftohf8_3(vector<3xf16>) -> vector<3xi8>
+// CHECK: llvm.func spir_funccc @_Z13convert_half3Dv3_f(vector<3xf32>) -> vector<3xf16>
+// CHECK: llvm.func spir_funccc @__builtin_IB_bftof_3(vector<3xi16>) -> vector<3xf32>
+// CHECK-LABEL: llvm.func @truncf_bf16_to_hf8_v3
+// CHECK-SAME: %[[ARG0:.*]]: vector<3xbf16>
+llvm.func @truncf_bf16_to_hf8_v3(%src: vector<3xbf16>) -> vector<3xi8> {
+ // CHECK: %[[BC:.*]] = llvm.bitcast %[[ARG0]] : vector<3xbf16> to vector<3xi16>
+ // CHECK: %[[F32:.*]] = llvm.call spir_funccc @__builtin_IB_bftof_3(%[[BC]])
+ // CHECK-SAME: : (vector<3xi16>) -> vector<3xf32>
+ // CHECK: %[[F16:.*]] = llvm.call spir_funccc @_Z13convert_half3Dv3_f(%[[F32]])
+ // CHECK-SAME: : (vector<3xf32>) -> vector<3xf16>
+ // CHECK: %[[RES:.*]] = llvm.call spir_funccc @__builtin_IB_hftohf8_3(%[[F16]])
+ // CHECK-SAME: : (vector<3xf16>) -> vector<3xi8>
+ %dst = xevm.truncf %src { src_etype = bf16, dst_etype = f8 } : (vector<3xbf16>) -> vector<3xi8>
+ llvm.return %dst : vector<3xi8>
+}
+
+// -----
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_hftobf8_8(vector<8xf16>) -> vector<8xi8>
+// CHECK-LABEL: llvm.func @truncf_f16_to_bf8_v8
+llvm.func @truncf_f16_to_bf8_v8(%src: vector<8xf16>) -> vector<8xi8> {
+ // CHECK: llvm.call spir_funccc @__builtin_IB_hftobf8_8({{.*}}) {{.*}} : (vector<8xf16>) -> vector<8xi8>
+ %dst = xevm.truncf %src { src_etype = f16, dst_etype = bf8 } : (vector<8xf16>) -> vector<8xi8>
+ llvm.return %dst : vector<8xi8>
+}
+
+// -----
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_bf8tohf_2(vector<2xi8>) -> vector<2xf16>
+// CHECK-LABEL: llvm.func @extf_bf8_to_f16_v2
+llvm.func @extf_bf8_to_f16_v2(%src: vector<2xi8>) -> vector<2xf16> {
+ // CHECK: llvm.call spir_funccc @__builtin_IB_bf8tohf_2({{.*}}) {{.*}} : (vector<2xi8>) -> vector<2xf16>
+ %dst = xevm.extf %src { src_etype = bf8, dst_etype = f16 } : (vector<2xi8>) -> vector<2xf16>
+ llvm.return %dst : vector<2xf16>
+}
+
+// -----
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_ftobf_4(vector<4xf32>) -> vector<4xi16>
+// CHECK: llvm.func spir_funccc @_Z14convert_float4Dv4_Dh(vector<4xf16>) -> vector<4xf32>
+// CHECK: llvm.func spir_funccc @__builtin_IB_bf8tohf_4(vector<4xi8>) -> vector<4xf16>
+// CHECK-LABEL: llvm.func @extf_bf8_to_bf16_v4
+// CHECK-SAME: %[[ARG0:.*]]: vector<4xi8>
+llvm.func @extf_bf8_to_bf16_v4(%src: vector<4xi8>) -> vector<4xbf16> {
+ // CHECK: %[[F16:.*]] = llvm.call spir_funccc @__builtin_IB_bf8tohf_4(%[[ARG0]])
+ // CHECK-SAME: : (vector<4xi8>) -> vector<4xf16>
+ // CHECK: %[[F32:.*]] = llvm.call spir_funccc @_Z14convert_float4Dv4_Dh(%[[F16]])
+ // CHECK-SAME: : (vector<4xf16>) -> vector<4xf32>
+ // CHECK: %[[BF:.*]] = llvm.call spir_funccc @__builtin_IB_ftobf_4(%[[F32]])
+ // CHECK-SAME: : (vector<4xf32>) -> vector<4xi16>
+ // CHECK: %[[RES:.*]] = llvm.bitcast %[[BF]] : vector<4xi16> to vector<4xbf16>
+ %dst = xevm.extf %src { src_etype = bf8, dst_etype = bf16 } : (vector<4xi8>) -> vector<4xbf16>
+ llvm.return %dst : vector<4xbf16>
+}
+
+// -----
+
+// Two fp4 values pack into a single byte. SPIR-V has no vector of length one,
+// so the packed side is a scalar, and only the low byte of the dnscl result is
+// kept.
+// CHECK: llvm.func spir_funccc @__builtin_IB_dnscl_hf16(i32, i32, i32, i32) -> i32
+// CHECK-LABEL: llvm.func @truncf_f16_to_e2m1_v2
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>
+llvm.func @truncf_f16_to_e2m1_v2(%src: vector<2xf16>) -> i8 {
+ // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+ // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK: %[[BC:.*]] = llvm.bitcast %[[ARG0]] : vector<2xf16> to i32
+ // CHECK: %[[CALL:.*]] = llvm.call spir_funccc @__builtin_IB_dnscl_hf16(%[[BC]], %[[UNDEF]], %[[C1]], %[[C0]])
+ // CHECK-SAME: : (i32, i32, i32, i32) -> i32
+ // CHECK: %[[RES:.*]] = llvm.trunc %[[CALL]] : i32 to i8
+ %dst = xevm.truncf %src { src_etype = f16, dst_etype = e2m1 } : (vector<2xf16>) -> i8
+ llvm.return %dst : i8
+}
+
+// -----
+
+// An element pair is the conversion granularity, so 3 elements are padded up to
+// 4 and the spare nibble is left undefined. The two bytes the call writes, 0
+// and 2, are compacted into the result.
+// CHECK: llvm.func spir_funccc @__builtin_IB_dnscl_hf16(i32, i32, i32, i32) -> i32
+// CHECK-LABEL: llvm.func @truncf_f16_to_e2m1_v3
+// CHECK-SAME: %[[ARG0:.*]]: vector<3xf16>
+llvm.func @truncf_f16_to_e2m1_v3(%src: vector<3xf16>) -> vector<2xi8> {
+ // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK: %[[PAD:.*]] = llvm.shufflevector %[[ARG0]], %[[ARG0]] [0, 1, 2, 0] : vector<3xf16>
+ // CHECK: %[[LANES:.*]] = llvm.bitcast %[[PAD]] : vector<4xf16> to vector<2xi32>
+ // CHECK-DAG: %[[L1:.*]] = llvm.extractelement %[[LANES]][%[[C1]] : i32] : vector<2xi32>
+ // CHECK-DAG: %[[L0:.*]] = llvm.extractelement %[[LANES]][%[[C0]] : i32] : vector<2xi32>
+ // CHECK: %[[CALL:.*]] = llvm.call spir_funccc @__builtin_IB_dnscl_hf16(%[[L0]], %[[L1]], %[[C1]], %[[C0]])
+ // CHECK-SAME: : (i32, i32, i32, i32) -> i32
+ // CHECK: %[[BYTES:.*]] = llvm.bitcast %[[CALL]] : i32 to vector<4xi8>
+ // CHECK: %[[RES:.*]] = llvm.shufflevector %[[BYTES]], %[[BYTES]] [0, 2] : vector<4xi8>
+ %dst = xevm.truncf %src { src_etype = f16, dst_etype = e2m1 } : (vector<3xf16>) -> vector<2xi8>
+ llvm.return %dst : vector<2xi8>
+}
+
+// -----
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_dnscl_hf16(i32, i32, i32, i32) -> i32
+// CHECK-LABEL: llvm.func @truncf_f16_to_e2m1_v4
+// CHECK-SAME: %[[ARG0:.*]]: vector<4xf16>
+llvm.func @truncf_f16_to_e2m1_v4(%src: vector<4xf16>) -> vector<2xi8> {
+ // CHECK: %[[LANES:.*]] = llvm.bitcast %[[ARG0]] : vector<4xf16> to vector<2xi32>
+ // CHECK: llvm.call spir_funccc @__builtin_IB_dnscl_hf16
+ // CHECK: %[[BYTES:.*]] = llvm.bitcast {{.*}} : i32 to vector<4xi8>
+ // CHECK: %[[RES:.*]] = llvm.shufflevector %[[BYTES]], %[[BYTES]] [0, 2] : vector<4xi8>
+ %dst = xevm.truncf %src { src_etype = f16, dst_etype = e2m1 } : (vector<4xf16>) -> vector<2xi8>
+ llvm.return %dst : vector<2xi8>
+}
+
+// -----
+
+// Eight elements fill a dword: two calls with complementary nibble modes, 0 for
+// bytes 0 and 2 and 2 for bytes 1 and 3, are OR-ed together.
+// CHECK: llvm.func spir_funccc @__builtin_IB_dnscl_bf16(i32, i32, i32, i32) -> i32
+// CHECK-LABEL: llvm.func @truncf_bf16_to_e2m1_v8
+// CHECK-SAME: %[[ARG0:.*]]: vector<8xbf16>
+llvm.func @truncf_bf16_to_e2m1_v8(%src: vector<8xbf16>) -> vector<4xi8> {
+ // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C2:.*]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C3:.*]] = llvm.mlir.constant(3 : i32) : i32
+ // CHECK: %[[LANES:.*]] = llvm.bitcast %[[ARG0]] : vector<8xbf16> to vector<4xi32>
+ // CHECK: %[[L0:.*]] = llvm.extractelement %[[LANES]][%[[C0]] : i32] : vector<4xi32>
+ // CHECK: %[[L2:.*]] = llvm.extractelement %[[LANES]][%[[C2]] : i32] : vector<4xi32>
+ // CHECK: %[[EVEN:.*]] = llvm.call spir_funccc @__builtin_IB_dnscl_bf16(%[[L0]], %[[L2]], %[[C1]], %[[C0]])
+ // CHECK-SAME: : (i32, i32, i32, i32) -> i32
+ // CHECK: %[[L1:.*]] = llvm.extractelement %[[LANES]][%[[C1]] : i32] : vector<4xi32>
+ // CHECK: %[[L3:.*]] = llvm.extractelement %[[LANES]][%[[C3]] : i32] : vector<4xi32>
+ // CHECK: %[[ODD:.*]] = llvm.call spir_funccc @__builtin_IB_dnscl_bf16(%[[L1]], %[[L3]], %[[C1]], %[[C2]])
+ // CHECK-SAME: : (i32, i32, i32, i32) -> i32
+ // CHECK: %[[OR:.*]] = llvm.or %[[EVEN]], %[[ODD]] : i32
+ // CHECK: %[[RES:.*]] = llvm.bitcast %[[OR]] : i32 to vector<4xi8>
+ %dst = xevm.truncf %src { src_etype = bf16, dst_etype = e2m1 } : (vector<8xbf16>) -> vector<4xi8>
+ llvm.return %dst : vector<4xi8>
+}
+
+// -----
+
+// The fp4 up-conversion takes a scalar source byte for the shortest length.
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_to_fp16_packed(vector<16xi32>, i8) -> i32
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_lut(i32) -> vector<16xi32>
+// CHECK-LABEL: llvm.func @extf_e2m1_to_f16_v2
+// CHECK-SAME: %[[ARG0:.*]]: i8
+llvm.func @extf_e2m1_to_f16_v2(%src: i8) -> vector<2xf16> {
+ // CHECK: %[[LUTIDX:.*]] = llvm.mlir.constant(7 : i32) : i32
+ // CHECK: %[[LUT:.*]] = llvm.call spir_funccc @__builtin_IB_shfl_idx4_lut(%[[LUTIDX]])
+ // CHECK-SAME: : (i32) -> vector<16xi32>
+ // CHECK: %[[CONV:.*]] = llvm.call spir_funccc @__builtin_IB_shfl_idx4_to_fp16_packed(%[[LUT]], %[[ARG0]])
+ // CHECK-SAME: : (vector<16xi32>, i8) -> i32
+ // CHECK: %[[RES:.*]] = llvm.bitcast %[[CONV]] : i32 to vector<2xf16>
+ %dst = xevm.extf %src { src_etype = e2m1, dst_etype = f16 } : (i8) -> vector<2xf16>
+ llvm.return %dst : vector<2xf16>
+}
+
+// -----
+
+// Three values are read as two whole bytes, and the spare value dropped.
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_to_fp16_2_packed(vector<16xi32>, vector<2xi8>) -> vector<2xi32>
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_lut(i32) -> vector<16xi32>
+// CHECK-LABEL: llvm.func @extf_e2m1_to_f16_v3
+// CHECK-SAME: %[[ARG0:.*]]: vector<2xi8>
+llvm.func @extf_e2m1_to_f16_v3(%src: vector<2xi8>) -> vector<3xf16> {
+ // CHECK: %[[CONV:.*]] = llvm.call spir_funccc @__builtin_IB_shfl_idx4_to_fp16_2_packed({{.*}}, %[[ARG0]])
+ // CHECK-SAME: : (vector<16xi32>, vector<2xi8>) -> vector<2xi32>
+ // CHECK: %[[WIDE:.*]] = llvm.bitcast %[[CONV]] : vector<2xi32> to vector<4xf16>
+ // CHECK: %[[RES:.*]] = llvm.shufflevector %[[WIDE]], %[[WIDE]] [0, 1, 2] : vector<4xf16>
+ %dst = xevm.extf %src { src_etype = e2m1, dst_etype = f16 } : (vector<2xi8>) -> vector<3xf16>
+ llvm.return %dst : vector<3xf16>
+}
+
+// -----
+
+// The lookup table index selects the destination format: 5 for bf16, 7 for f16.
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_to_fp16_2_packed(vector<16xi32>, vector<2xi8>) -> vector<2xi32>
+// CHECK-LABEL: llvm.func @extf_e2m1_to_bf16_v4
+llvm.func @extf_e2m1_to_bf16_v4(%src: vector<2xi8>) -> vector<4xbf16> {
+ // CHECK: %[[LUTIDX:.*]] = llvm.mlir.constant(5 : i32) : i32
+ // CHECK: llvm.call spir_funccc @__builtin_IB_shfl_idx4_lut(%[[LUTIDX]])
+ // CHECK: %[[CONV:.*]] = llvm.call spir_funccc @__builtin_IB_shfl_idx4_to_fp16_2_packed
+ // CHECK: %[[RES:.*]] = llvm.bitcast %[[CONV]] : vector<2xi32> to vector<4xbf16>
+ %dst = xevm.extf %src { src_etype = e2m1, dst_etype = bf16 } : (vector<2xi8>) -> vector<4xbf16>
+ llvm.return %dst : vector<4xbf16>
+}
+
+// -----
+
+// CHECK: llvm.func spir_funccc @__builtin_IB_shfl_idx4_to_fp16_4_packed(vector<16xi32>, vector<4xi8>) -> vector<4xi32>
+// CHECK-LABEL: llvm.func @extf_e2m1_to_f16_v8
+// CHECK-SAME: %[[ARG0:.*]]: vector<4xi8>
+llvm.func @extf_e2m1_to_f16_v8(%src: vector<4xi8>) -> vector<8xf16> {
+ // CHECK: %[[CONV:.*]] = llvm.call spir_funccc @__builtin_IB_shfl_idx4_to_fp16_4_packed({{.*}}, %[[ARG0]])
+ // CHECK-SAME: : (vector<16xi32>, vector<4xi8>) -> vector<4xi32>
+ // CHECK: %[[RES:.*]] = llvm.bitcast %[[CONV]] : vector<4xi32> to vector<8xf16>
+ %dst = xevm.extf %src { src_etype = e2m1, dst_etype = f16 } : (vector<4xi8>) -> vector<8xf16>
+ llvm.return %dst : vector<8xf16>
+}
diff --git a/mlir/test/Conversion/XeVMToLLVM/xevm_mx-to-llvm.mlir b/mlir/test/Conversion/XeVMToLLVM/xevm_mx-to-llvm.mlir
index 2ac76ac1e73a0..bc92009292959 100644
--- a/mlir/test/Conversion/XeVMToLLVM/xevm_mx-to-llvm.mlir
+++ b/mlir/test/Conversion/XeVMToLLVM/xevm_mx-to-llvm.mlir
@@ -84,15 +84,15 @@ llvm.func @truncf_bf16_to_hf8(%src: vector<16xbf16>) -> vector<16xi8> {
// CHECK-LABEL: llvm.func @truncf_f16_to_e2m1
// CHECK-SAME: %[[ARG0:.*]]: vector<16xf16>
llvm.func @truncf_f16_to_e2m1(%src: vector<16xf16>) -> vector<8xi8> {
- // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : vector<2xi32>
- // CHECK: %[[C7:.*]] = llvm.mlir.constant(7 : i32) : i32
- // CHECK: %[[C6:.*]] = llvm.mlir.constant(6 : i32) : i32
- // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : i32) : i32
- // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : i32) : i32
- // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : i32) : i32
- // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : i32) : i32
- // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
- // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[UNDEF:.*]] = llvm.mlir.undef : vector<2xi32>
+ // CHECK-DAG: %[[C7:.*]] = llvm.mlir.constant(7 : i32) : i32
+ // CHECK-DAG: %[[C6:.*]] = llvm.mlir.constant(6 : i32) : i32
+ // CHECK-DAG: %[[C5:.*]] = llvm.mlir.constant(5 : i32) : i32
+ // CHECK-DAG: %[[C4:.*]] = llvm.mlir.constant(4 : i32) : i32
+ // CHECK-DAG: %[[C3:.*]] = llvm.mlir.constant(3 : i32) : i32
+ // CHECK-DAG: %[[C2:.*]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
// CHECK: %[[BC:.*]] = llvm.bitcast %[[ARG0]] : vector<16xf16> to vector<8xi32>
// CHECK: %[[E0:.*]] = llvm.extractelement %[[BC]][%[[C0]] : i32] : vector<8xi32>
// CHECK: %[[E2:.*]] = llvm.extractelement %[[BC]][%[[C2]] : i32] : vector<8xi32>
@@ -125,15 +125,15 @@ llvm.func @truncf_f16_to_e2m1(%src: vector<16xf16>) -> vector<8xi8> {
// CHECK-LABEL: llvm.func @truncf_bf16_to_e2m1
// CHECK-SAME: %[[ARG0:.*]]: vector<16xbf16>
llvm.func @truncf_bf16_to_e2m1(%src: vector<16xbf16>) -> vector<8xi8> {
- // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : vector<2xi32>
- // CHECK: %[[C7:.*]] = llvm.mlir.constant(7 : i32) : i32
- // CHECK: %[[C6:.*]] = llvm.mlir.constant(6 : i32) : i32
- // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : i32) : i32
- // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : i32) : i32
- // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : i32) : i32
- // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : i32) : i32
- // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
- // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[UNDEF:.*]] = llvm.mlir.undef : vector<2xi32>
+ // CHECK-DAG: %[[C7:.*]] = llvm.mlir.constant(7 : i32) : i32
+ // CHECK-DAG: %[[C6:.*]] = llvm.mlir.constant(6 : i32) : i32
+ // CHECK-DAG: %[[C5:.*]] = llvm.mlir.constant(5 : i32) : i32
+ // CHECK-DAG: %[[C4:.*]] = llvm.mlir.constant(4 : i32) : i32
+ // CHECK-DAG: %[[C3:.*]] = llvm.mlir.constant(3 : i32) : i32
+ // CHECK-DAG: %[[C2:.*]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
// CHECK: %[[BC:.*]] = llvm.bitcast %[[ARG0]] : vector<16xbf16> to vector<8xi32>
// CHECK: %[[E0:.*]] = llvm.extractelement %[[BC]][%[[C0]] : i32] : vector<8xi32>
// CHECK: %[[E2:.*]] = llvm.extractelement %[[BC]][%[[C2]] : i32] : vector<8xi32>
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index cf91beb67a96c..5293021622020 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -2058,7 +2058,7 @@ llvm.func @invalid_xevm_matrix_3(%a: !llvm.ptr<1>, %base_width_a: i32, %base_hei
// -----
llvm.func @invalid_xevm_truncf_1(%arg0: vector<8xf16>) {
- // expected-error at +1 {{op both src and dst should be vector types or both}}
+ // expected-error at +1 {{op dst should be 64 bits wide to hold 8 value(s) of 8 bits, but it is 8}}
%0 = xevm.truncf %arg0 { src_etype = f16, dst_etype = bf8 } : (vector<8xf16>) -> i8
llvm.return
}
@@ -2066,7 +2066,7 @@ llvm.func @invalid_xevm_truncf_1(%arg0: vector<8xf16>) {
// -----
llvm.func @invalid_xevm_truncf_2(%arg0: f16) {
- // expected-error at +1 {{op both src and dst should be vector types or both}}
+ // expected-error at +1 {{op dst should be 8 bits wide to hold 1 value(s) of 8 bits, but it is 64}}
%0 = xevm.truncf %arg0 { src_etype = f16, dst_etype = bf8 } : (f16) -> vector<8xi8>
llvm.return
}
@@ -2074,7 +2074,7 @@ llvm.func @invalid_xevm_truncf_2(%arg0: f16) {
// -----
llvm.func @invalid_xevm_extf_1(%arg0: vector<8xi8>) {
- // expected-error at +1 {{op both src and dst should be vector types or both}}
+ // expected-error at +1 {{op src should be 8 bits wide to hold 1 value(s) of 8 bits, but it is 64}}
%0 = xevm.extf %arg0 { src_etype = bf8, dst_etype = f16 } : (vector<8xi8>) -> f16
llvm.return
}
@@ -2082,7 +2082,7 @@ llvm.func @invalid_xevm_extf_1(%arg0: vector<8xi8>) {
// -----
llvm.func @invalid_xevm_extf_2(%arg0: i8) {
- // expected-error at +1 {{op both src and dst should be vector types or both}}
+ // expected-error at +1 {{op src should be 64 bits wide to hold 8 value(s) of 8 bits, but it is 8}}
%0 = xevm.extf %arg0 { src_etype = bf8, dst_etype = f16 } : (i8) -> vector<8xf16>
llvm.return
}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
new file mode 100644
index 0000000000000..28c44b8ca3245
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
@@ -0,0 +1,192 @@
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN: | mlir-runner \
+// RUN: --shared-libs=%mlir_levelzero_runtime \
+// RUN: --shared-libs=%mlir_runner_utils \
+// RUN: --shared-libs=%mlir_c_runner_utils \
+// RUN: --entry-point-result=void \
+// RUN: | FileCheck %s
+
+// XFAIL:*
+// Round trip test for xevm.truncf followed by xevm.extf with the fp8 (bf8,
+// f8E5M2) format and a bf16 source and destination, at the SPIR-V vector
+// lengths other than 16, which xevm_truncf_extf_roundtrip_bf16.mlir covers.
+//
+// A bf16 operand is converted through f32 and f16 rather than directly, so this
+// exercises __builtin_IB_bftof_N and convert_halfN on the way down, and
+// convert_floatN and __builtin_IB_ftobf_N on the way back up, at each length.
+//
+// Each of the 16 lanes owns one row of 32 bf16 values and converts four slices
+// of it, of 2, 3, 4 and 8 elements. Every value used is exactly representable in
+// bf8, which has two mantissa bits (0, 0.5, 1, 1.5, 2, 3, 4, 6), so each round
+// trip must reproduce its input. One fp8 value occupies a whole byte, so the
+// packed type has the same length as the source, including the 3 element case
+// that IGC provides a builtin for.
+//
+// Each slice starts at a 16 byte aligned offset in the row, at elements 0, 8, 16
+// and 24, so the gaps between them are never written. Results go to a second
+// buffer pre-filled with -1, so the gaps read back as -1 and a conversion that
+// wrote nothing would leave -1 where a value is expected.
+module @roundtrip attributes {gpu.container_module} {
+
+ gpu.module @kernel {
+ gpu.func @roundtrip_bf16_lengths(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel {
+ %lane = gpu.lane_id
+ %lane_i64 = arith.index_cast %lane : index to i64
+ %row_len = arith.constant 32 : i64
+ %row = arith.muli %lane_i64, %row_len : i64
+ %src_row = llvm.getelementptr %src[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst_row = llvm.getelementptr %dst[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+
+ %c8 = arith.constant 8 : i64
+ %c16 = arith.constant 16 : i64
+ %c24 = arith.constant 24 : i64
+
+ // 2 elements.
+ %v2 = llvm.load %src_row : !llvm.ptr<1> -> vector<2xbf16>
+ %t2 = xevm.truncf %v2 { src_etype = bf16, dst_etype = bf8 }
+ : (vector<2xbf16>) -> vector<2xi8>
+ %e2 = xevm.extf %t2 { src_etype = bf8, dst_etype = bf16 }
+ : (vector<2xi8>) -> vector<2xbf16>
+ llvm.store %e2, %dst_row : vector<2xbf16>, !llvm.ptr<1>
+
+ // 3 elements, a length the fp8 builtins provide directly.
+ %src3 = llvm.getelementptr %src_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst3 = llvm.getelementptr %dst_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v3 = llvm.load %src3 : !llvm.ptr<1> -> vector<3xbf16>
+ %t3 = xevm.truncf %v3 { src_etype = bf16, dst_etype = bf8 }
+ : (vector<3xbf16>) -> vector<3xi8>
+ %e3 = xevm.extf %t3 { src_etype = bf8, dst_etype = bf16 }
+ : (vector<3xi8>) -> vector<3xbf16>
+ llvm.store %e3, %dst3 : vector<3xbf16>, !llvm.ptr<1>
+
+ // 4 elements.
+ %src4 = llvm.getelementptr %src_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst4 = llvm.getelementptr %dst_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v4 = llvm.load %src4 : !llvm.ptr<1> -> vector<4xbf16>
+ %t4 = xevm.truncf %v4 { src_etype = bf16, dst_etype = bf8 }
+ : (vector<4xbf16>) -> vector<4xi8>
+ %e4 = xevm.extf %t4 { src_etype = bf8, dst_etype = bf16 }
+ : (vector<4xi8>) -> vector<4xbf16>
+ llvm.store %e4, %dst4 : vector<4xbf16>, !llvm.ptr<1>
+
+ // 8 elements.
+ %src8 = llvm.getelementptr %src_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst8 = llvm.getelementptr %dst_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v8 = llvm.load %src8 : !llvm.ptr<1> -> vector<8xbf16>
+ %t8 = xevm.truncf %v8 { src_etype = bf16, dst_etype = bf8 }
+ : (vector<8xbf16>) -> vector<8xi8>
+ %e8 = xevm.extf %t8 { src_etype = bf8, dst_etype = bf16 }
+ : (vector<8xi8>) -> vector<8xbf16>
+ llvm.store %e8, %dst8 : vector<8xbf16>, !llvm.ptr<1>
+
+ gpu.return
+ }
+ }
+
+ func.func @test(%src : memref<16x32xbf16>, %dst : memref<16x32xbf16>) -> memref<16x32xbf16>
+ attributes {llvm.emit_c_interface} {
+ %c1 = arith.constant 1 : index
+ %c16 = arith.constant 16 : index
+ %dev_src = gpu.alloc() : memref<16x32xbf16>
+ %dev_dst = gpu.alloc() : memref<16x32xbf16>
+ gpu.memcpy %dev_src, %src : memref<16x32xbf16>, memref<16x32xbf16>
+ gpu.memcpy %dev_dst, %dst : memref<16x32xbf16>, memref<16x32xbf16>
+ %s0 = memref.extract_aligned_pointer_as_index %dev_src : memref<16x32xbf16> -> index
+ %s1 = arith.index_cast %s0 : index to i64
+ %s2 = llvm.inttoptr %s1 : i64 to !llvm.ptr
+ %src_casted = llvm.addrspacecast %s2 : !llvm.ptr to !llvm.ptr<1>
+ %d0 = memref.extract_aligned_pointer_as_index %dev_dst : memref<16x32xbf16> -> index
+ %d1 = arith.index_cast %d0 : index to i64
+ %d2 = llvm.inttoptr %d1 : i64 to !llvm.ptr
+ %dst_casted = llvm.addrspacecast %d2 : !llvm.ptr to !llvm.ptr<1>
+ gpu.launch_func @kernel::@roundtrip_bf16_lengths blocks in (%c1, %c1, %c1)
+ threads in (%c16, %c1, %c1)
+ args(%src_casted : !llvm.ptr<1>, %dst_casted : !llvm.ptr<1>)
+ %out = memref.alloc() : memref<16x32xbf16>
+ gpu.memcpy %out, %dev_dst : memref<16x32xbf16>, memref<16x32xbf16>
+ gpu.dealloc %dev_src : memref<16x32xbf16>
+ gpu.dealloc %dev_dst : memref<16x32xbf16>
+ return %out : memref<16x32xbf16>
+ }
+
+ func.func @main() attributes {llvm.emit_c_interface} {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c3 = arith.constant 3 : index
+ %c4 = arith.constant 4 : index
+ %c5 = arith.constant 5 : index
+ %c6 = arith.constant 6 : index
+ %c7 = arith.constant 7 : index
+ %c8 = arith.constant 8 : index
+ %c16 = arith.constant 16 : index
+ %c32 = arith.constant 32 : index
+
+ // Lookup table of 8 magnitudes exactly representable in bf8.
+ %lut = memref.alloc() : memref<8xbf16>
+ %v0 = arith.constant 0.0 : bf16
+ %v1 = arith.constant 0.5 : bf16
+ %v2 = arith.constant 1.0 : bf16
+ %v3 = arith.constant 1.5 : bf16
+ %v4 = arith.constant 2.0 : bf16
+ %v5 = arith.constant 3.0 : bf16
+ %v6 = arith.constant 4.0 : bf16
+ %v7 = arith.constant 6.0 : bf16
+ memref.store %v0, %lut[%c0] : memref<8xbf16>
+ memref.store %v1, %lut[%c1] : memref<8xbf16>
+ memref.store %v2, %lut[%c2] : memref<8xbf16>
+ memref.store %v3, %lut[%c3] : memref<8xbf16>
+ memref.store %v4, %lut[%c4] : memref<8xbf16>
+ memref.store %v5, %lut[%c5] : memref<8xbf16>
+ memref.store %v6, %lut[%c6] : memref<8xbf16>
+ memref.store %v7, %lut[%c7] : memref<8xbf16>
+
+ // Source rows repeat the representable values, so each slice starts at 0 and
+ // the 8 element slice covers the whole set.
+ %A = memref.alloc() : memref<16x32xbf16>
+ %B = memref.alloc() : memref<16x32xbf16>
+ %sentinel = arith.constant -1.0 : bf16
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %jm8 = arith.remui %j, %c8 : index
+ %val = memref.load %lut[%jm8] : memref<8xbf16>
+ memref.store %val, %A[%i, %j] : memref<16x32xbf16>
+ memref.store %sentinel, %B[%i, %j] : memref<16x32xbf16>
+ }
+ }
+
+ %C = call @test(%A, %B) : (memref<16x32xbf16>, memref<16x32xbf16>) -> memref<16x32xbf16>
+
+ // Convert the bf16 result to f32 so it can be printed with printMemrefF32.
+ %Cf32 = memref.alloc() : memref<16x32xf32>
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %v = memref.load %C[%i, %j] : memref<16x32xbf16>
+ %vf = arith.extf %v : bf16 to f32
+ memref.store %vf, %Cf32[%i, %j] : memref<16x32xf32>
+ }
+ }
+ %C_cast = memref.cast %Cf32 : memref<16x32xf32> to memref<*xf32>
+ call @printMemrefF32(%C_cast) : (memref<*xf32>) -> ()
+
+ // The four converted slices sit at elements 0, 8, 16 and 24. Everything else
+ // keeps the -1 the destination was filled with.
+ // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ memref.dealloc %A : memref<16x32xbf16>
+ memref.dealloc %B : memref<16x32xbf16>
+ memref.dealloc %C : memref<16x32xbf16>
+ memref.dealloc %Cf32 : memref<16x32xf32>
+ memref.dealloc %lut : memref<8xbf16>
+ return
+ }
+ func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
new file mode 100644
index 0000000000000..46a09c0a1a004
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
@@ -0,0 +1,192 @@
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN: | mlir-runner \
+// RUN: --shared-libs=%mlir_levelzero_runtime \
+// RUN: --shared-libs=%mlir_runner_utils \
+// RUN: --shared-libs=%mlir_c_runner_utils \
+// RUN: --entry-point-result=void \
+// RUN: | FileCheck %s
+
+// XFAIL:*
+// Round trip test for xevm.truncf followed by xevm.extf with the fp4 (e2m1)
+// format and a bf16 source and destination, at the SPIR-V vector lengths other
+// than 16, which xevm_truncf_extf_roundtrip_fp4_bf16.mlir covers.
+//
+// bf16 needs no conversion through f32 here, unlike the fp8 case: the down
+// conversion has its own __builtin_IB_dnscl_bf16, and the up conversion selects
+// bf16 with a different lookup table index.
+//
+// Each of the 16 lanes owns one row of 32 bf16 values and converts four slices
+// of it, of 2, 3, 4 and 8 elements. Every value used is exactly representable in
+// e2m1 (0, 0.5, 1, 1.5, 2, 3, 4, 6), so each round trip must reproduce its
+// input. The packed widths differ per slice: 2 values pack into a single byte,
+// which SPIR-V spells as a scalar, 3 and 4 into vector<2xi8>, and 8 into
+// vector<4xi8>.
+//
+// Each slice starts at a 16 byte aligned offset in the row, at elements 0, 8, 16
+// and 24, so the gaps between them are never written. Results go to a second
+// buffer pre-filled with -1, so the gaps read back as -1 and a conversion that
+// wrote nothing would leave -1 where a value is expected.
+module @roundtrip attributes {gpu.container_module} {
+
+ gpu.module @kernel {
+ gpu.func @roundtrip_fp4_bf16_lengths(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel {
+ %lane = gpu.lane_id
+ %lane_i64 = arith.index_cast %lane : index to i64
+ %row_len = arith.constant 32 : i64
+ %row = arith.muli %lane_i64, %row_len : i64
+ %src_row = llvm.getelementptr %src[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst_row = llvm.getelementptr %dst[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+
+ %c8 = arith.constant 8 : i64
+ %c16 = arith.constant 16 : i64
+ %c24 = arith.constant 24 : i64
+
+ // 2 elements, packed into one byte, so the packed value is a scalar.
+ %v2 = llvm.load %src_row : !llvm.ptr<1> -> vector<2xbf16>
+ %t2 = xevm.truncf %v2 { src_etype = bf16, dst_etype = e2m1 }
+ : (vector<2xbf16>) -> i8
+ %e2 = xevm.extf %t2 { src_etype = e2m1, dst_etype = bf16 }
+ : (i8) -> vector<2xbf16>
+ llvm.store %e2, %dst_row : vector<2xbf16>, !llvm.ptr<1>
+
+ // 3 elements, padded up to a whole pair, leaving one spare nibble.
+ %src3 = llvm.getelementptr %src_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst3 = llvm.getelementptr %dst_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v3 = llvm.load %src3 : !llvm.ptr<1> -> vector<3xbf16>
+ %t3 = xevm.truncf %v3 { src_etype = bf16, dst_etype = e2m1 }
+ : (vector<3xbf16>) -> vector<2xi8>
+ %e3 = xevm.extf %t3 { src_etype = e2m1, dst_etype = bf16 }
+ : (vector<2xi8>) -> vector<3xbf16>
+ llvm.store %e3, %dst3 : vector<3xbf16>, !llvm.ptr<1>
+
+ // 4 elements, one call whose two written bytes are compacted.
+ %src4 = llvm.getelementptr %src_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst4 = llvm.getelementptr %dst_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v4 = llvm.load %src4 : !llvm.ptr<1> -> vector<4xbf16>
+ %t4 = xevm.truncf %v4 { src_etype = bf16, dst_etype = e2m1 }
+ : (vector<4xbf16>) -> vector<2xi8>
+ %e4 = xevm.extf %t4 { src_etype = e2m1, dst_etype = bf16 }
+ : (vector<2xi8>) -> vector<4xbf16>
+ llvm.store %e4, %dst4 : vector<4xbf16>, !llvm.ptr<1>
+
+ // 8 elements, the point at which two calls fill a whole dword.
+ %src8 = llvm.getelementptr %src_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %dst8 = llvm.getelementptr %dst_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, bf16
+ %v8 = llvm.load %src8 : !llvm.ptr<1> -> vector<8xbf16>
+ %t8 = xevm.truncf %v8 { src_etype = bf16, dst_etype = e2m1 }
+ : (vector<8xbf16>) -> vector<4xi8>
+ %e8 = xevm.extf %t8 { src_etype = e2m1, dst_etype = bf16 }
+ : (vector<4xi8>) -> vector<8xbf16>
+ llvm.store %e8, %dst8 : vector<8xbf16>, !llvm.ptr<1>
+
+ gpu.return
+ }
+ }
+
+ func.func @test(%src : memref<16x32xbf16>, %dst : memref<16x32xbf16>) -> memref<16x32xbf16>
+ attributes {llvm.emit_c_interface} {
+ %c1 = arith.constant 1 : index
+ %c16 = arith.constant 16 : index
+ %dev_src = gpu.alloc() : memref<16x32xbf16>
+ %dev_dst = gpu.alloc() : memref<16x32xbf16>
+ gpu.memcpy %dev_src, %src : memref<16x32xbf16>, memref<16x32xbf16>
+ gpu.memcpy %dev_dst, %dst : memref<16x32xbf16>, memref<16x32xbf16>
+ %s0 = memref.extract_aligned_pointer_as_index %dev_src : memref<16x32xbf16> -> index
+ %s1 = arith.index_cast %s0 : index to i64
+ %s2 = llvm.inttoptr %s1 : i64 to !llvm.ptr
+ %src_casted = llvm.addrspacecast %s2 : !llvm.ptr to !llvm.ptr<1>
+ %d0 = memref.extract_aligned_pointer_as_index %dev_dst : memref<16x32xbf16> -> index
+ %d1 = arith.index_cast %d0 : index to i64
+ %d2 = llvm.inttoptr %d1 : i64 to !llvm.ptr
+ %dst_casted = llvm.addrspacecast %d2 : !llvm.ptr to !llvm.ptr<1>
+ gpu.launch_func @kernel::@roundtrip_fp4_bf16_lengths blocks in (%c1, %c1, %c1)
+ threads in (%c16, %c1, %c1)
+ args(%src_casted : !llvm.ptr<1>, %dst_casted : !llvm.ptr<1>)
+ %out = memref.alloc() : memref<16x32xbf16>
+ gpu.memcpy %out, %dev_dst : memref<16x32xbf16>, memref<16x32xbf16>
+ gpu.dealloc %dev_src : memref<16x32xbf16>
+ gpu.dealloc %dev_dst : memref<16x32xbf16>
+ return %out : memref<16x32xbf16>
+ }
+
+ func.func @main() attributes {llvm.emit_c_interface} {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c3 = arith.constant 3 : index
+ %c4 = arith.constant 4 : index
+ %c5 = arith.constant 5 : index
+ %c6 = arith.constant 6 : index
+ %c7 = arith.constant 7 : index
+ %c8 = arith.constant 8 : index
+ %c16 = arith.constant 16 : index
+ %c32 = arith.constant 32 : index
+
+ // Lookup table of the 8 magnitudes exactly representable in e2m1.
+ %lut = memref.alloc() : memref<8xbf16>
+ %v0 = arith.constant 0.0 : bf16
+ %v1 = arith.constant 0.5 : bf16
+ %v2 = arith.constant 1.0 : bf16
+ %v3 = arith.constant 1.5 : bf16
+ %v4 = arith.constant 2.0 : bf16
+ %v5 = arith.constant 3.0 : bf16
+ %v6 = arith.constant 4.0 : bf16
+ %v7 = arith.constant 6.0 : bf16
+ memref.store %v0, %lut[%c0] : memref<8xbf16>
+ memref.store %v1, %lut[%c1] : memref<8xbf16>
+ memref.store %v2, %lut[%c2] : memref<8xbf16>
+ memref.store %v3, %lut[%c3] : memref<8xbf16>
+ memref.store %v4, %lut[%c4] : memref<8xbf16>
+ memref.store %v5, %lut[%c5] : memref<8xbf16>
+ memref.store %v6, %lut[%c6] : memref<8xbf16>
+ memref.store %v7, %lut[%c7] : memref<8xbf16>
+
+ // Source rows repeat the representable values, so each slice starts at 0 and
+ // the 8 element slice covers the whole set.
+ %A = memref.alloc() : memref<16x32xbf16>
+ %B = memref.alloc() : memref<16x32xbf16>
+ %sentinel = arith.constant -1.0 : bf16
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %jm8 = arith.remui %j, %c8 : index
+ %val = memref.load %lut[%jm8] : memref<8xbf16>
+ memref.store %val, %A[%i, %j] : memref<16x32xbf16>
+ memref.store %sentinel, %B[%i, %j] : memref<16x32xbf16>
+ }
+ }
+
+ %C = call @test(%A, %B) : (memref<16x32xbf16>, memref<16x32xbf16>) -> memref<16x32xbf16>
+
+ // Convert the bf16 result to f32 so it can be printed with printMemrefF32.
+ %Cf32 = memref.alloc() : memref<16x32xf32>
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %v = memref.load %C[%i, %j] : memref<16x32xbf16>
+ %vf = arith.extf %v : bf16 to f32
+ memref.store %vf, %Cf32[%i, %j] : memref<16x32xf32>
+ }
+ }
+ %C_cast = memref.cast %Cf32 : memref<16x32xf32> to memref<*xf32>
+ call @printMemrefF32(%C_cast) : (memref<*xf32>) -> ()
+
+ // The four converted slices sit at elements 0, 8, 16 and 24. Everything else
+ // keeps the -1 the destination was filled with.
+ // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ memref.dealloc %A : memref<16x32xbf16>
+ memref.dealloc %B : memref<16x32xbf16>
+ memref.dealloc %C : memref<16x32xbf16>
+ memref.dealloc %Cf32 : memref<16x32xf32>
+ memref.dealloc %lut : memref<8xbf16>
+ return
+ }
+ func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
new file mode 100644
index 0000000000000..5b7171b20a4ed
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
@@ -0,0 +1,188 @@
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN: | mlir-runner \
+// RUN: --shared-libs=%mlir_levelzero_runtime \
+// RUN: --shared-libs=%mlir_runner_utils \
+// RUN: --shared-libs=%mlir_c_runner_utils \
+// RUN: --entry-point-result=void \
+// RUN: | FileCheck %s
+
+// XFAIL:*
+// Round trip test for xevm.truncf followed by xevm.extf with the fp4 (e2m1)
+// format, at the SPIR-V vector lengths other than 16, which
+// xevm_truncf_extf_roundtrip_fp4.mlir covers.
+//
+// Each of the 16 lanes owns one row of 32 f16 values and converts four slices of
+// it, of 2, 3, 4 and 8 elements. Every value used is exactly representable in
+// e2m1 (0, 0.5, 1, 1.5, 2, 3, 4, 6), so each round trip must reproduce its
+// input. The packed widths differ per slice: 2 values pack into a single byte,
+// which SPIR-V spells as a scalar, 3 and 4 into vector<2xi8>, and 8 into
+// vector<4xi8>.
+//
+// Each slice starts at a 16 byte aligned offset in the row, at elements 0, 8, 16
+// and 24, so the gaps between them are never written. Results go to a second
+// buffer pre-filled with -1, so the gaps read back as -1 and a conversion that
+// wrote nothing would leave -1 where a value is expected.
+module @roundtrip attributes {gpu.container_module} {
+
+ gpu.module @kernel {
+ gpu.func @roundtrip_fp4_lengths(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel {
+ %lane = gpu.lane_id
+ %lane_i64 = arith.index_cast %lane : index to i64
+ %row_len = arith.constant 32 : i64
+ %row = arith.muli %lane_i64, %row_len : i64
+ %src_row = llvm.getelementptr %src[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst_row = llvm.getelementptr %dst[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+
+ %c8 = arith.constant 8 : i64
+ %c16 = arith.constant 16 : i64
+ %c24 = arith.constant 24 : i64
+
+ // 2 elements, packed into one byte, so the packed value is a scalar.
+ %v2 = llvm.load %src_row : !llvm.ptr<1> -> vector<2xf16>
+ %t2 = xevm.truncf %v2 { src_etype = f16, dst_etype = e2m1 }
+ : (vector<2xf16>) -> i8
+ %e2 = xevm.extf %t2 { src_etype = e2m1, dst_etype = f16 }
+ : (i8) -> vector<2xf16>
+ llvm.store %e2, %dst_row : vector<2xf16>, !llvm.ptr<1>
+
+ // 3 elements, padded up to a whole pair, leaving one spare nibble.
+ %src3 = llvm.getelementptr %src_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst3 = llvm.getelementptr %dst_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v3 = llvm.load %src3 : !llvm.ptr<1> -> vector<3xf16>
+ %t3 = xevm.truncf %v3 { src_etype = f16, dst_etype = e2m1 }
+ : (vector<3xf16>) -> vector<2xi8>
+ %e3 = xevm.extf %t3 { src_etype = e2m1, dst_etype = f16 }
+ : (vector<2xi8>) -> vector<3xf16>
+ llvm.store %e3, %dst3 : vector<3xf16>, !llvm.ptr<1>
+
+ // 4 elements, one call whose two written bytes are compacted.
+ %src4 = llvm.getelementptr %src_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst4 = llvm.getelementptr %dst_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v4 = llvm.load %src4 : !llvm.ptr<1> -> vector<4xf16>
+ %t4 = xevm.truncf %v4 { src_etype = f16, dst_etype = e2m1 }
+ : (vector<4xf16>) -> vector<2xi8>
+ %e4 = xevm.extf %t4 { src_etype = e2m1, dst_etype = f16 }
+ : (vector<2xi8>) -> vector<4xf16>
+ llvm.store %e4, %dst4 : vector<4xf16>, !llvm.ptr<1>
+
+ // 8 elements, the point at which two calls fill a whole dword.
+ %src8 = llvm.getelementptr %src_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst8 = llvm.getelementptr %dst_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v8 = llvm.load %src8 : !llvm.ptr<1> -> vector<8xf16>
+ %t8 = xevm.truncf %v8 { src_etype = f16, dst_etype = e2m1 }
+ : (vector<8xf16>) -> vector<4xi8>
+ %e8 = xevm.extf %t8 { src_etype = e2m1, dst_etype = f16 }
+ : (vector<4xi8>) -> vector<8xf16>
+ llvm.store %e8, %dst8 : vector<8xf16>, !llvm.ptr<1>
+
+ gpu.return
+ }
+ }
+
+ func.func @test(%src : memref<16x32xf16>, %dst : memref<16x32xf16>) -> memref<16x32xf16>
+ attributes {llvm.emit_c_interface} {
+ %c1 = arith.constant 1 : index
+ %c16 = arith.constant 16 : index
+ %dev_src = gpu.alloc() : memref<16x32xf16>
+ %dev_dst = gpu.alloc() : memref<16x32xf16>
+ gpu.memcpy %dev_src, %src : memref<16x32xf16>, memref<16x32xf16>
+ gpu.memcpy %dev_dst, %dst : memref<16x32xf16>, memref<16x32xf16>
+ %s0 = memref.extract_aligned_pointer_as_index %dev_src : memref<16x32xf16> -> index
+ %s1 = arith.index_cast %s0 : index to i64
+ %s2 = llvm.inttoptr %s1 : i64 to !llvm.ptr
+ %src_casted = llvm.addrspacecast %s2 : !llvm.ptr to !llvm.ptr<1>
+ %d0 = memref.extract_aligned_pointer_as_index %dev_dst : memref<16x32xf16> -> index
+ %d1 = arith.index_cast %d0 : index to i64
+ %d2 = llvm.inttoptr %d1 : i64 to !llvm.ptr
+ %dst_casted = llvm.addrspacecast %d2 : !llvm.ptr to !llvm.ptr<1>
+ gpu.launch_func @kernel::@roundtrip_fp4_lengths blocks in (%c1, %c1, %c1)
+ threads in (%c16, %c1, %c1)
+ args(%src_casted : !llvm.ptr<1>, %dst_casted : !llvm.ptr<1>)
+ %out = memref.alloc() : memref<16x32xf16>
+ gpu.memcpy %out, %dev_dst : memref<16x32xf16>, memref<16x32xf16>
+ gpu.dealloc %dev_src : memref<16x32xf16>
+ gpu.dealloc %dev_dst : memref<16x32xf16>
+ return %out : memref<16x32xf16>
+ }
+
+ func.func @main() attributes {llvm.emit_c_interface} {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c3 = arith.constant 3 : index
+ %c4 = arith.constant 4 : index
+ %c5 = arith.constant 5 : index
+ %c6 = arith.constant 6 : index
+ %c7 = arith.constant 7 : index
+ %c8 = arith.constant 8 : index
+ %c16 = arith.constant 16 : index
+ %c32 = arith.constant 32 : index
+
+ // Lookup table of the 8 magnitudes exactly representable in e2m1.
+ %lut = memref.alloc() : memref<8xf16>
+ %v0 = arith.constant 0.0 : f16
+ %v1 = arith.constant 0.5 : f16
+ %v2 = arith.constant 1.0 : f16
+ %v3 = arith.constant 1.5 : f16
+ %v4 = arith.constant 2.0 : f16
+ %v5 = arith.constant 3.0 : f16
+ %v6 = arith.constant 4.0 : f16
+ %v7 = arith.constant 6.0 : f16
+ memref.store %v0, %lut[%c0] : memref<8xf16>
+ memref.store %v1, %lut[%c1] : memref<8xf16>
+ memref.store %v2, %lut[%c2] : memref<8xf16>
+ memref.store %v3, %lut[%c3] : memref<8xf16>
+ memref.store %v4, %lut[%c4] : memref<8xf16>
+ memref.store %v5, %lut[%c5] : memref<8xf16>
+ memref.store %v6, %lut[%c6] : memref<8xf16>
+ memref.store %v7, %lut[%c7] : memref<8xf16>
+
+ // Source rows repeat the representable values, so each slice starts at 0 and
+ // the 8 element slice covers the whole set.
+ %A = memref.alloc() : memref<16x32xf16>
+ %B = memref.alloc() : memref<16x32xf16>
+ %sentinel = arith.constant -1.0 : f16
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %jm8 = arith.remui %j, %c8 : index
+ %val = memref.load %lut[%jm8] : memref<8xf16>
+ memref.store %val, %A[%i, %j] : memref<16x32xf16>
+ memref.store %sentinel, %B[%i, %j] : memref<16x32xf16>
+ }
+ }
+
+ %C = call @test(%A, %B) : (memref<16x32xf16>, memref<16x32xf16>) -> memref<16x32xf16>
+
+ // Convert the f16 result to f32 so it can be printed with printMemrefF32.
+ %Cf32 = memref.alloc() : memref<16x32xf32>
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %v = memref.load %C[%i, %j] : memref<16x32xf16>
+ %vf = arith.extf %v : f16 to f32
+ memref.store %vf, %Cf32[%i, %j] : memref<16x32xf32>
+ }
+ }
+ %C_cast = memref.cast %Cf32 : memref<16x32xf32> to memref<*xf32>
+ call @printMemrefF32(%C_cast) : (memref<*xf32>) -> ()
+
+ // The four converted slices sit at elements 0, 8, 16 and 24. Everything else
+ // keeps the -1 the destination was filled with.
+ // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ memref.dealloc %A : memref<16x32xf16>
+ memref.dealloc %B : memref<16x32xf16>
+ memref.dealloc %C : memref<16x32xf16>
+ memref.dealloc %Cf32 : memref<16x32xf32>
+ memref.dealloc %lut : memref<8xf16>
+ return
+ }
+ func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
new file mode 100644
index 0000000000000..38ad7f0f1c2ef
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
@@ -0,0 +1,188 @@
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN: | mlir-runner \
+// RUN: --shared-libs=%mlir_levelzero_runtime \
+// RUN: --shared-libs=%mlir_runner_utils \
+// RUN: --shared-libs=%mlir_c_runner_utils \
+// RUN: --entry-point-result=void \
+// RUN: | FileCheck %s
+
+// XFAIL:*
+// Round trip test for xevm.truncf followed by xevm.extf with the fp8 (bf8,
+// f8E5M2) format, at the SPIR-V vector lengths other than 16, which
+// xevm_truncf_extf_roundtrip.mlir covers.
+//
+// Each of the 16 lanes owns one row of 32 f16 values and converts four slices of
+// it, of 2, 3, 4 and 8 elements. Every value used is exactly representable in
+// bf8, which has two mantissa bits (0, 0.5, 1, 1.5, 2, 3, 4, 6), so each round
+// trip must reproduce its input. One fp8 value occupies a whole byte, so the
+// packed type has the same length as the source, including the 3 element case
+// that IGC provides a builtin for.
+//
+// Each slice starts at a 16 byte aligned offset in the row, at elements 0, 8, 16
+// and 24, so the gaps between them are never written. Results go to a second
+// buffer pre-filled with -1, so the gaps read back as -1 and a conversion that
+// wrote nothing would leave -1 where a value is expected.
+module @roundtrip attributes {gpu.container_module} {
+
+ gpu.module @kernel {
+ gpu.func @roundtrip_fp8_lengths(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel {
+ %lane = gpu.lane_id
+ %lane_i64 = arith.index_cast %lane : index to i64
+ %row_len = arith.constant 32 : i64
+ %row = arith.muli %lane_i64, %row_len : i64
+ %src_row = llvm.getelementptr %src[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst_row = llvm.getelementptr %dst[%row]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+
+ %c8 = arith.constant 8 : i64
+ %c16 = arith.constant 16 : i64
+ %c24 = arith.constant 24 : i64
+
+ // 2 elements.
+ %v2 = llvm.load %src_row : !llvm.ptr<1> -> vector<2xf16>
+ %t2 = xevm.truncf %v2 { src_etype = f16, dst_etype = bf8 }
+ : (vector<2xf16>) -> vector<2xi8>
+ %e2 = xevm.extf %t2 { src_etype = bf8, dst_etype = f16 }
+ : (vector<2xi8>) -> vector<2xf16>
+ llvm.store %e2, %dst_row : vector<2xf16>, !llvm.ptr<1>
+
+ // 3 elements, a length the fp8 builtins provide directly.
+ %src3 = llvm.getelementptr %src_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst3 = llvm.getelementptr %dst_row[%c8]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v3 = llvm.load %src3 : !llvm.ptr<1> -> vector<3xf16>
+ %t3 = xevm.truncf %v3 { src_etype = f16, dst_etype = bf8 }
+ : (vector<3xf16>) -> vector<3xi8>
+ %e3 = xevm.extf %t3 { src_etype = bf8, dst_etype = f16 }
+ : (vector<3xi8>) -> vector<3xf16>
+ llvm.store %e3, %dst3 : vector<3xf16>, !llvm.ptr<1>
+
+ // 4 elements.
+ %src4 = llvm.getelementptr %src_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst4 = llvm.getelementptr %dst_row[%c16]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v4 = llvm.load %src4 : !llvm.ptr<1> -> vector<4xf16>
+ %t4 = xevm.truncf %v4 { src_etype = f16, dst_etype = bf8 }
+ : (vector<4xf16>) -> vector<4xi8>
+ %e4 = xevm.extf %t4 { src_etype = bf8, dst_etype = f16 }
+ : (vector<4xi8>) -> vector<4xf16>
+ llvm.store %e4, %dst4 : vector<4xf16>, !llvm.ptr<1>
+
+ // 8 elements.
+ %src8 = llvm.getelementptr %src_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %dst8 = llvm.getelementptr %dst_row[%c24]
+ : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, f16
+ %v8 = llvm.load %src8 : !llvm.ptr<1> -> vector<8xf16>
+ %t8 = xevm.truncf %v8 { src_etype = f16, dst_etype = bf8 }
+ : (vector<8xf16>) -> vector<8xi8>
+ %e8 = xevm.extf %t8 { src_etype = bf8, dst_etype = f16 }
+ : (vector<8xi8>) -> vector<8xf16>
+ llvm.store %e8, %dst8 : vector<8xf16>, !llvm.ptr<1>
+
+ gpu.return
+ }
+ }
+
+ func.func @test(%src : memref<16x32xf16>, %dst : memref<16x32xf16>) -> memref<16x32xf16>
+ attributes {llvm.emit_c_interface} {
+ %c1 = arith.constant 1 : index
+ %c16 = arith.constant 16 : index
+ %dev_src = gpu.alloc() : memref<16x32xf16>
+ %dev_dst = gpu.alloc() : memref<16x32xf16>
+ gpu.memcpy %dev_src, %src : memref<16x32xf16>, memref<16x32xf16>
+ gpu.memcpy %dev_dst, %dst : memref<16x32xf16>, memref<16x32xf16>
+ %s0 = memref.extract_aligned_pointer_as_index %dev_src : memref<16x32xf16> -> index
+ %s1 = arith.index_cast %s0 : index to i64
+ %s2 = llvm.inttoptr %s1 : i64 to !llvm.ptr
+ %src_casted = llvm.addrspacecast %s2 : !llvm.ptr to !llvm.ptr<1>
+ %d0 = memref.extract_aligned_pointer_as_index %dev_dst : memref<16x32xf16> -> index
+ %d1 = arith.index_cast %d0 : index to i64
+ %d2 = llvm.inttoptr %d1 : i64 to !llvm.ptr
+ %dst_casted = llvm.addrspacecast %d2 : !llvm.ptr to !llvm.ptr<1>
+ gpu.launch_func @kernel::@roundtrip_fp8_lengths blocks in (%c1, %c1, %c1)
+ threads in (%c16, %c1, %c1)
+ args(%src_casted : !llvm.ptr<1>, %dst_casted : !llvm.ptr<1>)
+ %out = memref.alloc() : memref<16x32xf16>
+ gpu.memcpy %out, %dev_dst : memref<16x32xf16>, memref<16x32xf16>
+ gpu.dealloc %dev_src : memref<16x32xf16>
+ gpu.dealloc %dev_dst : memref<16x32xf16>
+ return %out : memref<16x32xf16>
+ }
+
+ func.func @main() attributes {llvm.emit_c_interface} {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+ %c3 = arith.constant 3 : index
+ %c4 = arith.constant 4 : index
+ %c5 = arith.constant 5 : index
+ %c6 = arith.constant 6 : index
+ %c7 = arith.constant 7 : index
+ %c8 = arith.constant 8 : index
+ %c16 = arith.constant 16 : index
+ %c32 = arith.constant 32 : index
+
+ // Lookup table of 8 magnitudes exactly representable in bf8.
+ %lut = memref.alloc() : memref<8xf16>
+ %v0 = arith.constant 0.0 : f16
+ %v1 = arith.constant 0.5 : f16
+ %v2 = arith.constant 1.0 : f16
+ %v3 = arith.constant 1.5 : f16
+ %v4 = arith.constant 2.0 : f16
+ %v5 = arith.constant 3.0 : f16
+ %v6 = arith.constant 4.0 : f16
+ %v7 = arith.constant 6.0 : f16
+ memref.store %v0, %lut[%c0] : memref<8xf16>
+ memref.store %v1, %lut[%c1] : memref<8xf16>
+ memref.store %v2, %lut[%c2] : memref<8xf16>
+ memref.store %v3, %lut[%c3] : memref<8xf16>
+ memref.store %v4, %lut[%c4] : memref<8xf16>
+ memref.store %v5, %lut[%c5] : memref<8xf16>
+ memref.store %v6, %lut[%c6] : memref<8xf16>
+ memref.store %v7, %lut[%c7] : memref<8xf16>
+
+ // Source rows repeat the representable values, so each slice starts at 0 and
+ // the 8 element slice covers the whole set.
+ %A = memref.alloc() : memref<16x32xf16>
+ %B = memref.alloc() : memref<16x32xf16>
+ %sentinel = arith.constant -1.0 : f16
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %jm8 = arith.remui %j, %c8 : index
+ %val = memref.load %lut[%jm8] : memref<8xf16>
+ memref.store %val, %A[%i, %j] : memref<16x32xf16>
+ memref.store %sentinel, %B[%i, %j] : memref<16x32xf16>
+ }
+ }
+
+ %C = call @test(%A, %B) : (memref<16x32xf16>, memref<16x32xf16>) -> memref<16x32xf16>
+
+ // Convert the f16 result to f32 so it can be printed with printMemrefF32.
+ %Cf32 = memref.alloc() : memref<16x32xf32>
+ scf.for %i = %c0 to %c16 step %c1 {
+ scf.for %j = %c0 to %c32 step %c1 {
+ %v = memref.load %C[%i, %j] : memref<16x32xf16>
+ %vf = arith.extf %v : f16 to f32
+ memref.store %vf, %Cf32[%i, %j] : memref<16x32xf32>
+ }
+ }
+ %C_cast = memref.cast %Cf32 : memref<16x32xf32> to memref<*xf32>
+ call @printMemrefF32(%C_cast) : (memref<*xf32>) -> ()
+
+ // The four converted slices sit at elements 0, 8, 16 and 24. Everything else
+ // keeps the -1 the destination was filled with.
+ // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ memref.dealloc %A : memref<16x32xf16>
+ memref.dealloc %B : memref<16x32xf16>
+ memref.dealloc %C : memref<16x32xf16>
+ memref.dealloc %Cf32 : memref<16x32xf32>
+ memref.dealloc %lut : memref<8xf16>
+ return
+ }
+ func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+}
>From 53ab49ceb7d2b13a145368ae69f6ab58e582c812 Mon Sep 17 00:00:00 2001
From: "Lee, Sang Ik" <sang.ik.lee at intel.com>
Date: Wed, 26 Aug 2026 19:18:55 +0000
Subject: [PATCH 2/2] Check the lowering in the new fp conversion integration
tests instead of XFAIL
The execution part and its value checks are kept as RUN-DISABLED lines with an
EXEC check prefix.
---
...vm_truncf_extf_roundtrip_bf16_lengths.mlir | 23 ++++++++++---------
...runcf_extf_roundtrip_fp4_bf16_lengths.mlir | 23 ++++++++++---------
...evm_truncf_extf_roundtrip_fp4_lengths.mlir | 23 ++++++++++---------
...evm_truncf_extf_roundtrip_fp8_lengths.mlir | 23 ++++++++++---------
4 files changed, 48 insertions(+), 44 deletions(-)
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
index 28c44b8ca3245..81366afd50098 100644
--- a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_bf16_lengths.mlir
@@ -1,12 +1,13 @@
-// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
-// RUN: | mlir-runner \
-// RUN: --shared-libs=%mlir_levelzero_runtime \
-// RUN: --shared-libs=%mlir_runner_utils \
-// RUN: --shared-libs=%mlir_c_runner_utils \
-// RUN: --entry-point-result=void \
-// RUN: | FileCheck %s
-
-// XFAIL:*
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri"
+
+// RUN-DISABLED: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN-DISABLED: | mlir-runner \
+// RUN-DISABLED: --shared-libs=%mlir_levelzero_runtime \
+// RUN-DISABLED: --shared-libs=%mlir_runner_utils \
+// RUN-DISABLED: --shared-libs=%mlir_c_runner_utils \
+// RUN-DISABLED: --entry-point-result=void \
+// RUN-DISABLED: | FileCheck %s --check-prefix=EXEC
+
// Round trip test for xevm.truncf followed by xevm.extf with the fp8 (bf8,
// f8E5M2) format and a bf16 source and destination, at the SPIR-V vector
// lengths other than 16, which xevm_truncf_extf_roundtrip_bf16.mlir covers.
@@ -179,8 +180,8 @@ module @roundtrip attributes {gpu.container_module} {
// The four converted slices sit at elements 0, 8, 16 and 24. Everything else
// keeps the -1 the destination was filled with.
- // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
- // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ // EXEC: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // EXEC-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
memref.dealloc %A : memref<16x32xbf16>
memref.dealloc %B : memref<16x32xbf16>
memref.dealloc %C : memref<16x32xbf16>
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
index 46a09c0a1a004..dc5e7595fd8aa 100644
--- a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_bf16_lengths.mlir
@@ -1,12 +1,13 @@
-// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
-// RUN: | mlir-runner \
-// RUN: --shared-libs=%mlir_levelzero_runtime \
-// RUN: --shared-libs=%mlir_runner_utils \
-// RUN: --shared-libs=%mlir_c_runner_utils \
-// RUN: --entry-point-result=void \
-// RUN: | FileCheck %s
-
-// XFAIL:*
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri"
+
+// RUN-DISABLED: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN-DISABLED: | mlir-runner \
+// RUN-DISABLED: --shared-libs=%mlir_levelzero_runtime \
+// RUN-DISABLED: --shared-libs=%mlir_runner_utils \
+// RUN-DISABLED: --shared-libs=%mlir_c_runner_utils \
+// RUN-DISABLED: --entry-point-result=void \
+// RUN-DISABLED: | FileCheck %s --check-prefix=EXEC
+
// Round trip test for xevm.truncf followed by xevm.extf with the fp4 (e2m1)
// format and a bf16 source and destination, at the SPIR-V vector lengths other
// than 16, which xevm_truncf_extf_roundtrip_fp4_bf16.mlir covers.
@@ -179,8 +180,8 @@ module @roundtrip attributes {gpu.container_module} {
// The four converted slices sit at elements 0, 8, 16 and 24. Everything else
// keeps the -1 the destination was filled with.
- // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
- // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ // EXEC: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // EXEC-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
memref.dealloc %A : memref<16x32xbf16>
memref.dealloc %B : memref<16x32xbf16>
memref.dealloc %C : memref<16x32xbf16>
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
index 5b7171b20a4ed..11dfd21c4dc92 100644
--- a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp4_lengths.mlir
@@ -1,12 +1,13 @@
-// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
-// RUN: | mlir-runner \
-// RUN: --shared-libs=%mlir_levelzero_runtime \
-// RUN: --shared-libs=%mlir_runner_utils \
-// RUN: --shared-libs=%mlir_c_runner_utils \
-// RUN: --entry-point-result=void \
-// RUN: | FileCheck %s
-
-// XFAIL:*
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri"
+
+// RUN-DISABLED: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN-DISABLED: | mlir-runner \
+// RUN-DISABLED: --shared-libs=%mlir_levelzero_runtime \
+// RUN-DISABLED: --shared-libs=%mlir_runner_utils \
+// RUN-DISABLED: --shared-libs=%mlir_c_runner_utils \
+// RUN-DISABLED: --entry-point-result=void \
+// RUN-DISABLED: | FileCheck %s --check-prefix=EXEC
+
// Round trip test for xevm.truncf followed by xevm.extf with the fp4 (e2m1)
// format, at the SPIR-V vector lengths other than 16, which
// xevm_truncf_extf_roundtrip_fp4.mlir covers.
@@ -175,8 +176,8 @@ module @roundtrip attributes {gpu.container_module} {
// The four converted slices sit at elements 0, 8, 16 and 24. Everything else
// keeps the -1 the destination was filled with.
- // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
- // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ // EXEC: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // EXEC-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
memref.dealloc %A : memref<16x32xf16>
memref.dealloc %B : memref<16x32xf16>
memref.dealloc %C : memref<16x32xf16>
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
index 38ad7f0f1c2ef..aa8630181af5e 100644
--- a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_truncf_extf_roundtrip_fp8_lengths.mlir
@@ -1,12 +1,13 @@
-// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
-// RUN: | mlir-runner \
-// RUN: --shared-libs=%mlir_levelzero_runtime \
-// RUN: --shared-libs=%mlir_runner_utils \
-// RUN: --shared-libs=%mlir_c_runner_utils \
-// RUN: --entry-point-result=void \
-// RUN: | FileCheck %s
-
-// XFAIL:*
+// RUN: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri"
+
+// RUN-DISABLED: mlir-opt %s --gpu-lower-to-xevm-pipeline="xegpu-op-level=lane zebin-chip=cri" \
+// RUN-DISABLED: | mlir-runner \
+// RUN-DISABLED: --shared-libs=%mlir_levelzero_runtime \
+// RUN-DISABLED: --shared-libs=%mlir_runner_utils \
+// RUN-DISABLED: --shared-libs=%mlir_c_runner_utils \
+// RUN-DISABLED: --entry-point-result=void \
+// RUN-DISABLED: | FileCheck %s --check-prefix=EXEC
+
// Round trip test for xevm.truncf followed by xevm.extf with the fp8 (bf8,
// f8E5M2) format, at the SPIR-V vector lengths other than 16, which
// xevm_truncf_extf_roundtrip.mlir covers.
@@ -175,8 +176,8 @@ module @roundtrip attributes {gpu.container_module} {
// The four converted slices sit at elements 0, 8, 16 and 24. Everything else
// keeps the -1 the destination was filled with.
- // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
- // CHECK-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
+ // EXEC: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+ // EXEC-COUNT-16: [0, 0.5, -1, -1, -1, -1, -1, -1, 0, 0.5, 1, -1, -1, -1, -1, -1, 0, 0.5, 1, 1.5, -1, -1, -1, -1, 0, 0.5, 1, 1.5, 2, 3, 4, 6]
memref.dealloc %A : memref<16x32xf16>
memref.dealloc %B : memref<16x32xf16>
memref.dealloc %C : memref<16x32xf16>
More information about the Mlir-commits
mailing list