[Mlir-commits] [mlir] [MLIR][XeVM] XeVM to LLVM: Use fp conversion spirv builtins (PR #232463)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Thu Oct 15 12:02:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir
Author: Sang Ik Lee (silee2)
<details>
<summary>Changes</summary>
IGC's __builtin_IB API is not stable so move to __builtin_spirv if an alternative is available.
For fp4, this update results in reducing some scalarized instructions as __builtin_spirv API support vector operand directly.
---
Patch is 61.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/232463.diff
3 Files Affected:
- (modified) mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp (+137-258)
- (modified) mlir/test/Conversion/XeVMToLLVM/xevm_fp_conversion_lengths.mlir (+56-98)
- (modified) mlir/test/Conversion/XeVMToLLVM/xevm_mx-to-llvm.mlir (+40-130)
``````````diff
diff --git a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
index f3bf1a5fc7866c..d9989d74dcbdca 100644
--- a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
+++ b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
@@ -1177,19 +1177,6 @@ static Value castIfNeeded(ConversionPatternRewriter &rewriter, Location loc,
return LLVM::BitcastOp::create(rewriter, loc, ty, val);
}
-/// Selects `numElements` leading elements of the vector `val`. Used to drop the
-/// padding the 3 element case needs, as the hardware conversions work on whole
-/// pairs of elements.
-static Value takeLeadingElements(ConversionPatternRewriter &rewriter,
- Location loc, Value val, int64_t numElements) {
- auto vecTy = cast<VectorType>(val.getType());
- if (vecTy.getNumElements() == numElements)
- return val;
- SmallVector<int32_t> mask =
- llvm::to_vector(llvm::seq<int32_t>(0, static_cast<int32_t>(numElements)));
- return LLVM::ShuffleVectorOp::create(rewriter, loc, val, val, mask);
-}
-
//
// Note: TruncfToOCLPattern and ExtfToOCLPattern does not lower to OpenCL API
// calls as there are not official ones yet. They are lowered directly to Intel
@@ -1239,174 +1226,92 @@ class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
// Handle the case where dst type is fp4 first.
if (dstEtype == TruncfDstElemTypes::E2M1) {
- // `__builtin_IB_dnscl_{hf16,bf16}(uint a, uint b, convert_to, mode)`
- // takes two dwords, each holding two source elements, and packs each pair
- // into one byte of the result dword. `mode` picks which bytes of that
- // dword are written: mode 0 writes bytes 0 and 2, mode 2 writes bytes 1
- // and 3. Two calls with complementary modes therefore OR together into
- // one fully packed dword covering eight source elements.
+ // IGC's BiFModule exposes the fp4 packing as an overloaded SPIR-V
+ // friendly builtin, `__builtin_spirv_Convert{BF16,FP16}ToE2M1INTEL`,
+ // declared for every SPIR-V vector length. Its implementation *is* the
+ // `__builtin_IB_dnscl_*` sequence this used to emit by hand - pairs of
+ // calls with complementary byte modes, ORed together - so calling the
+ // vector form hands that breakdown to IGC. Doing it here required
+ // splitting the source into dword lanes, which cost two
+ // `llvm.extractelement` per `dnscl` call and left the backend a
+ // scalarised sequence to re-vectorise.
//
- // A pair of elements is the conversion granularity, so an odd length is
- // padded up and the spare nibble left undefined.
- constexpr int kDnsclConvertToE2M1 = 1;
- constexpr int kDnsclModeBytes02 = 0;
- constexpr int kDnsclModeBytes13 = 2;
- // One dword lane per element pair, and one result byte per lane. The op
- // verifier has already checked that the destination is exactly that wide.
- int64_t numLanes = llvm::divideCeil(numElements, 2);
-
- Type i32Ty = rewriter.getI32Type();
- Type i8Ty = rewriter.getI8Type();
- // Pad an odd length up to a whole number of pairs, then view the source
- // as dword lanes.
- Value padded = src;
- if (numElements != numLanes * 2) {
- SmallVector<int32_t> mask = llvm::to_vector(
- llvm::seq<int32_t>(0, static_cast<int32_t>(numElements)));
- // The padding element is never read back, so any valid index will do.
- mask.append(static_cast<size_t>(numLanes * 2 - numElements), 0);
- padded = LLVM::ShuffleVectorOp::create(rewriter, loc, src, src, mask);
- }
- // A single lane is passed as a bare i32 rather than a one element vector,
- // which SPIR-V has no type for.
- Value laneVec;
- if (numLanes > 1)
- laneVec = LLVM::BitcastOp::create(
- rewriter, loc, VectorType::get(numLanes, i32Ty), padded);
- else
- laneVec = LLVM::BitcastOp::create(rewriter, loc, i32Ty, padded);
- auto getLane = [&](int64_t idx) -> Value {
- if (numLanes == 1)
- return laneVec;
- Value pos =
- LLVM::ConstantOp::create(rewriter, loc, rewriter.getI32Type(), idx);
- return LLVM::ExtractElementOp::create(rewriter, loc, laneVec, pos)
- ->getResult(0);
- };
-
- std::string fnName = "__builtin_IB_dnscl_";
- fnName += (srcEtype == TruncfSrcElemTypes::F16) ? "hf16" : "bf16";
- Value convertTo =
- LLVM::ConstantOp::create(rewriter, loc, i32Ty, kDnsclConvertToE2M1);
- auto genDnscl = [&](Value lo, Value hi, int mode) -> Value {
- Value modeVal = LLVM::ConstantOp::create(rewriter, loc, i32Ty, mode);
- SmallVector<Type> argTypes{lo.getType(), hi.getType(),
- convertTo.getType(), modeVal.getType()};
- SmallVector<Value> args{lo, hi, convertTo, modeVal};
- return createDeviceFunctionCall(rewriter, fnName, i32Ty, argTypes, args,
- {}, funcAttrs, op.getOperation())
- ->getResult(0);
- };
-
- Value result;
- if (numLanes <= 2) {
- // Fewer than four lanes cannot fill a dword, so a single call is made
- // and the written bytes, 0 and 2, are compacted afterwards.
- Value lo = getLane(0);
- Value hi =
- numLanes == 2
- ? getLane(1)
- : LLVM::UndefOp::create(rewriter, loc, i32Ty)->getResult(0);
- Value dword = genDnscl(lo, hi, kDnsclModeBytes02);
- if (numLanes == 1) {
- // A single byte, so the low one, is all that is kept.
- result = LLVM::TruncOp::create(rewriter, loc, i8Ty, dword);
- } else {
- Value bytes = LLVM::BitcastOp::create(
- rewriter, loc, VectorType::get(4, i8Ty), dword);
- result = LLVM::ShuffleVectorOp::create(rewriter, loc, bytes, bytes,
- ArrayRef<int32_t>{0, 2});
- }
+ // See IGC/BiFModule/Implementation/SPV_INTEL_fp_conversions/
+ // downconversions_fp4.cl for the declarations and their semantics.
+ Value arg = src;
+ StringRef baseName;
+ if (srcEtype == TruncfSrcElemTypes::F16) {
+ baseName = "__builtin_spirv_ConvertFP16ToE2M1INTEL";
} else {
- // Four lanes, eight source elements, per fully packed dword.
- SmallVector<Value> dwords;
- for (int64_t base = 0; base < numLanes; base += 4) {
- // Each lane is bound to a name first: `getLane` builds ops, and the
- // order of evaluation within an argument list is unspecified, which
- // would otherwise leave the order of the emitted ops up to the host
- // compiler.
- Value lane0 = getLane(base);
- Value lane2 = getLane(base + 2);
- Value even = genDnscl(lane0, lane2, kDnsclModeBytes02);
- Value lane1 = getLane(base + 1);
- Value lane3 = getLane(base + 3);
- Value odd = genDnscl(lane1, lane3, kDnsclModeBytes13);
- dwords.push_back(LLVM::OrOp::create(rewriter, loc, even, odd));
- }
- if (dwords.size() == 1) {
- result = dwords.front();
- } else {
- Type packedTy = VectorType::get(dwords.size(), i32Ty);
- result = LLVM::UndefOp::create(rewriter, loc, packedTy);
- for (auto [idx, dword] : llvm::enumerate(dwords)) {
- Value pos = LLVM::ConstantOp::create(rewriter, loc, i32Ty, idx);
- result =
- LLVM::InsertElementOp::create(rewriter, loc, result, dword, pos)
- ->getResult(0);
- }
- }
+ baseName = "__builtin_spirv_ConvertBF16ToE2M1INTEL";
+ // bf16 is passed as its bit pattern, which the builtin spells `short`.
+ arg = LLVM::BitcastOp::create(
+ rewriter, loc,
+ VectorType::get(vecSrcTy.getShape(), rewriter.getI16Type()), src);
}
+ // A pair of elements is the conversion granularity and packs into one
+ // byte, so an odd length rounds up and leaves the spare nibble undefined.
+ // The builtin returns that as a vector even when it is a single byte
+ // (`uchar1`), which is what the op's destination type already is.
+ int64_t numBytes = llvm::divideCeil(numElements, 2);
+ Type retTy = VectorType::get(numBytes, rewriter.getI8Type());
+ SmallVector<Type> argTypes{arg.getType()};
+ SmallVector<Value> args{arg};
+ Value result = createDeviceFunctionCall(
+ rewriter, mangle(baseName, argTypes), retTy, argTypes,
+ args, {}, funcAttrs, op.getOperation())
+ ->getResult(0);
rewriter.replaceOp(op, castIfNeeded(rewriter, loc, dstTy, result));
return success();
}
// Handle the case where dst type is fp8.
- // The fp8 conversions come as one builtin per vector length, so the length
- // is simply appended to the builtin name.
- std::string lenSuffix = std::to_string(numElements);
- // BF16 type needs some preprocessing before conversion,
- // First extended to F32 and then truncated to F16.
- if (srcEtype == TruncfSrcElemTypes::BF16) {
- // Step 1: Extend to F32
- // Use floatN __builtin_IB_bftof_N(shortN)
+ //
+ // The conversions come from the SPV_INTEL_fp_conversions half of the IGC
+ // BiFModule, which overloads one wrapper per vector length:
+ // charN __builtin_spirv_ConvertFP16ToE5M2EXT(halfN)
+ // charN __builtin_spirv_ConvertFP16ToE4M3EXT(halfN)
+ // charN __builtin_spirv_ConvertBF16ToE5M2EXT(shortN)
+ // charN __builtin_spirv_ConvertBF16ToE4M3EXT(shortN)
+ //
+ // These wrap the __builtin_IB_* builtins that used to be called here. As
+ // the note above says, they are the experimental SPIR-V extension
+ // implementation rather than an official OpenCL API, so the TODO to move
+ // once one exists still stands. They are still preferable to naming the
+ // IGC-internal builtins directly: being declared `overloadable` they are
+ // mangled, so the vector length comes from the argument type instead of a
+ // suffix pasted onto the name, and a name or signature mismatch fails to
+ // link rather than binding silently to nothing and yielding undef. Note
+ // too that IGC spells E4M3 as `e4m3` in the conversion builtins rather
+ // than `hf8`, even though the scaled DPAS builtin does use `hf8`; going
+ // through the wrappers avoids having to track that asymmetry here.
+ //
+ // A bf16 source takes the bf16 form, which accepts the bits as shortN and
+ // routes through f32 and f16 internally, so no preprocessing is needed
+ // beyond the bitcast to shortN.
+ bool srcIsBF16 = srcEtype == TruncfSrcElemTypes::BF16;
+ if (srcIsBF16)
src = LLVM::BitcastOp::create(
rewriter, op.getLoc(),
VectorType::get(vecSrcTy.getShape(), rewriter.getI16Type()), src);
- std::string fnName = "__builtin_IB_bftof_" + lenSuffix;
- SmallVector<Type> argTypes{src.getType()};
- SmallVector<Value> args{src};
- Type resTy = VectorType::get(vecSrcTy.getShape(), rewriter.getF32Type());
- src = createDeviceFunctionCall(rewriter, fnName, resTy, argTypes, args,
- {}, funcAttrs, op.getOperation())
- ->getResult(0);
- // Step 2: Truncf to F16
- // Use halfN convert_halfN(floatN)
- std::string truncFnName = "convert_half" + lenSuffix;
- SmallVector<Type> truncArgTypes{src.getType()};
- SmallVector<Value> truncArgs{src};
- truncFnName = mangle(truncFnName, truncArgTypes);
- resTy = VectorType::get(vecSrcTy.getShape(), rewriter.getF16Type());
- src =
- createDeviceFunctionCall(rewriter, truncFnName, resTy, truncArgTypes,
- truncArgs, {}, funcAttrs, op.getOperation())
- ->getResult(0);
- }
+ StringRef convertName;
if (dstEtype == TruncfDstElemTypes::BF8) { // Float8E5M2Type
- // Use charN __builtin_IB_hftobf8_N(halfN)
- std::string fnName = "__builtin_IB_hftobf8_" + lenSuffix;
- SmallVector<Type> argTypes{src.getType()};
- SmallVector<Value> args{src};
- Value result =
- createDeviceFunctionCall(rewriter, fnName, dstTy, argTypes, args, {},
- funcAttrs, op.getOperation())
- ->getResult(0);
-
- rewriter.replaceOp(op, result);
+ convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE5M2EXT"
+ : "__builtin_spirv_ConvertFP16ToE5M2EXT";
} else if (dstEtype == TruncfDstElemTypes::F8) { // Float8E4M3FNType
- // Use charN __builtin_IB_hftohf8_N(halfN)
- std::string fnName = "__builtin_IB_hftohf8_" + lenSuffix;
- SmallVector<Type> argTypes{src.getType()};
- SmallVector<Value> args{src};
- Value result =
- createDeviceFunctionCall(rewriter, fnName, dstTy, argTypes, args, {},
- funcAttrs, op.getOperation())
- ->getResult(0);
-
- rewriter.replaceOp(op, result);
+ convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE4M3EXT"
+ : "__builtin_spirv_ConvertFP16ToE4M3EXT";
} else {
return rewriter.notifyMatchFailure(
op, "Unsupported src, dst element type pair.");
}
+ SmallVector<Type> argTypes{src.getType()};
+ SmallVector<Value> args{src};
+ Value result = createDeviceFunctionCall(
+ rewriter, mangle(convertName, argTypes), dstTy, argTypes,
+ args, {}, funcAttrs, op.getOperation())
+ ->getResult(0);
+ rewriter.replaceOp(op, result);
return success();
}
};
@@ -1446,58 +1351,50 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
// Handle the case where src type is fp4 (e2m1) first.
if (srcEtype == ExtfSrcElemTypes::E2M1) {
- // Two fp4 values are packed per source byte, and one builtin exists per
- // source byte count:
- // uint16 __builtin_IB_shfl_idx4_lut(int lut_index)
- // uint __builtin_IB_shfl_idx4_to_fp16_packed(uint16 lut, char src)
- // uintN __builtin_IB_shfl_idx4_to_fp16_N_packed(uint16 lut, charN src)
- // Each returns one dword, holding two f16/bf16 values, per source byte.
- // The lookup table selects the target format:
- // 7 = e2m1 -> f16, 5 = e2m1 -> bf16.
+ // IGC's BiFModule exposes the fp4 unpacking as
+ // `__builtin_spirv_ConvertE2M1To{FP16,BF16}INTEL`, declared for every
+ // SPIR-V vector length. Its implementation *is* the
+ // `__builtin_IB_shfl_idx4_*` pair this used to emit by hand - the format
+ // lookup table, the per-byte packed conversion, and the trailing trim an
+ // odd length needs - so calling it hands all of that to IGC.
+ //
+ // Unlike the other conversions here the name cannot come from mangle().
+ // These builtins take a vector of i4, which the SPIR-V translator
+ // mangles as `i`, the same code as `int`, and IGC's PromoteSubByte pass
+ // then widens the parameter to whole bytes. So the declared parameter is
+ // half as wide as the name says: `Dv<N>_i` takes `charN/2`.
+ // upconversions_fp4.cl mangles these by hand for the same reason, as
+ // `overloadable` would mangle them on `char` instead. Build the suffix
+ // from the element count to match.
//
- // A byte is the conversion granularity, so an odd length reads one spare
- // value that is dropped afterwards. The op verifier has already checked
- // that the source is exactly as wide as those bytes.
+ // See IGC/BiFModule/Implementation/SPV_INTEL_fp_conversions/
+ // upconversions_fp4.cl for the declarations and their semantics.
+ StringRef baseName = (dstEtype == ExtfDstElemTypes::F16)
+ ? "__builtin_spirv_ConvertE2M1ToFP16INTEL"
+ : "__builtin_spirv_ConvertE2M1ToBF16INTEL";
+ // Two fp4 values pack into one byte, so the source is half as wide,
+ // rounded up. A scalar source is spelled as such for a single byte, and
+ // the builtin wants the one element vector, hence the cast.
int64_t numBytes = llvm::divideCeil(numElements, 2);
- constexpr int kLutE2M1ToF16 = 7;
- constexpr int kLutE2M1ToBF16 = 5;
- int lutIndex =
- (dstEtype == ExtfDstElemTypes::F16) ? kLutE2M1ToF16 : kLutE2M1ToBF16;
- Value lutIdx = LLVM::ConstantOp::create(rewriter, loc,
- rewriter.getI32Type(), lutIndex);
- Type lutTy = VectorType::get(16, rewriter.getI32Type());
- Value lut =
- createDeviceFunctionCall(rewriter, "__builtin_IB_shfl_idx4_lut",
- lutTy, {lutIdx.getType()}, {lutIdx}, {},
- funcAttrs, op.getOperation())
- ->getResult(0);
- // A single byte is passed as a bare i8, and one dword returned as a bare
- // i32, rather than as one element vectors SPIR-V has no type for.
- Type i8Ty = rewriter.getI8Type();
- Type i32Ty = rewriter.getI32Type();
- std::string fnName = "__builtin_IB_shfl_idx4_to_fp16_";
- Type argTy, packedResTy;
- if (numBytes == 1) {
- argTy = i8Ty;
- packedResTy = i32Ty;
- } else {
- fnName += std::to_string(numBytes) + "_";
- argTy = VectorType::get(numBytes, i8Ty);
- packedResTy = VectorType::get(numBytes, i32Ty);
- }
- fnName += "packed";
- SmallVector<Type> convArgTypes{lut.getType(), argTy};
- SmallVector<Value> convArgs{lut, castIfNeeded(rewriter, loc, argTy, src)};
+ Value arg = castIfNeeded(
+ rewriter, loc, VectorType::get(numBytes, rewriter.getI8Type()), src);
+ // A bf16 result comes back as its bit pattern, which the builtin spells
+ // `short`. The builtin returns exactly `numElements` values, having
+ // already dropped what an odd length rounded up to.
+ Type resElemTy = (dstEtype == ExtfDstElemTypes::F16)
+ ? cast<Type>(rewriter.getF16Type())
+ : cast<Type>(rewriter.getI16Type());
+ Type resTy = VectorType::get(numElements, resElemTy);
+ std::string fnName = llvm::formatv("_Z{0}{1}Dv{2}_i", baseName.size(),
+ baseName, numElements)
+ .str();
+ SmallVector<Type> argTypes{arg.getType()};
+ SmallVector<Value> args{arg};
Value result =
- createDeviceFunctionCall(rewriter, fnName, packedResTy, convArgTypes,
- convArgs, {}, funcAttrs, op.getOperation())
+ createDeviceFunctionCall(rewriter, fnName, resTy, argTypes, args, {},
+ funcAttrs, op.getOperation())
->getResult(0);
- // The builtin returns the f16/bf16 bits packed as i32, bitcast to the
- // f16/bf16 dst type and drop the padding an odd length produced.
- Type wideTy = VectorType::get(numBytes * 2, vecDstTy.getElementType());
- result = LLVM::BitcastOp::create(rewriter, loc, wideTy, result);
- result = takeLeadingElements(rewriter, loc, result, numElements);
- rewriter.replaceOp(op, result);
+ rewriter.replaceOp(op, castIfNeeded(rewriter, loc, vecDstTy, result));
return success();
}
@@ -1507,54 +1404,36 @@ class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
if (!vecSrcTy || vecSrcTy.getNumElements() != numElements)
return rewriter.notifyMatchFailure(
op, "fp8 src and dst must have the same number of elements");
- std::string lenSuffix = std::to_string(numElements);
-
- // Step 1: Extend fp8 (bf8/hf8) to F16.
- // bf8 -> half: halfN __builtin_IB_bf8tohf_N(charN)
- // hf8 -> half: halfN __builtin_IB_hf8tohf_N(charN)
- std::string fnName = (srcEtype == ExtfSrcElemTypes::BF8)
- ? "__builtin_IB_bf8tohf_"
- : "__builtin_IB_hf8tohf_";
- fnName += lenSuffix;
- Type f16Ty = VectorType::get(vecSrcTy.getShape(), rewriter.getF16Type());
+ // Mangled SPV_INTEL_fp_conversions wr...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/232463
More information about the Mlir-commits
mailing list