[llvm] [mlir] [mlir][x86vector] Simplify intrinsic generation (PR #133692)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 3 00:26:50 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir
Author: Adam Siemieniuk (adam-smnk)
<details>
<summary>Changes</summary>
Replaces separate x86vector named intrinsic operations with direct calls to LLVM intrinsic functions.
This rework reduces the number of named ops leaving only high-level MLIR equivalents of whole intrinsic classes e.g., variants of AVX512 dot on BF16 inputs. Dialect conversion applies LLVM intrinsic name mangling further simplifying lowering logic.
The separate conversion step translating x86vector intrinsics into LLVM IR is also eliminated. Instead, this step is now performed by the existing llvm dialect infrastructure.
RFC: https://discourse.llvm.org/t/rfc-simplify-x86-intrinsic-generation/85581
---
Patch is 48.61 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/133692.diff
13 Files Affected:
- (modified) mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt (-4)
- (modified) mlir/include/mlir/Dialect/X86Vector/X86Vector.td (+84-139)
- (modified) mlir/include/mlir/Target/LLVMIR/Dialect/All.h (-2)
- (removed) mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h (-32)
- (modified) mlir/lib/Dialect/X86Vector/Transforms/CMakeLists.txt (-3)
- (modified) mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp (+58-185)
- (modified) mlir/lib/Target/LLVMIR/CMakeLists.txt (-1)
- (modified) mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt (-1)
- (removed) mlir/lib/Target/LLVMIR/Dialect/X86Vector/CMakeLists.txt (-16)
- (removed) mlir/lib/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.cpp (-58)
- (modified) mlir/test/Dialect/X86Vector/legalize-for-llvm.mlir (+32-24)
- (modified) mlir/test/Target/LLVMIR/x86vector.mlir (+54-59)
- (modified) utils/bazel/llvm-project-overlay/mlir/BUILD.bazel (-13)
``````````diff
diff --git a/mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt b/mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt
index a22f8332514b9..498fa7e7d1b44 100644
--- a/mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/X86Vector/CMakeLists.txt
@@ -1,6 +1,2 @@
add_mlir_dialect(X86Vector x86vector)
add_mlir_doc(X86Vector X86Vector Dialects/ -gen-dialect-doc -dialect=x86vector)
-
-set(LLVM_TARGET_DEFINITIONS X86Vector.td)
-mlir_tablegen(X86VectorConversions.inc -gen-llvmir-conversions)
-add_public_tablegen_target(MLIRX86VectorConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/X86Vector/X86Vector.td b/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
index 566013e73f4b8..5b37d92600c9e 100644
--- a/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
+++ b/mlir/include/mlir/Dialect/X86Vector/X86Vector.td
@@ -34,25 +34,6 @@ def X86Vector_Dialect : Dialect {
class AVX512_Op<string mnemonic, list<Trait> traits = []> :
Op<X86Vector_Dialect, "avx512." # mnemonic, traits> {}
-// Intrinsic operation used during lowering to LLVM IR.
-class AVX512_IntrOp<string mnemonic, int numResults,
- list<Trait> traits = [],
- string extension = ""> :
- LLVM_IntrOpBase<X86Vector_Dialect, "avx512.intr." # mnemonic,
- !subst("EXT", extension, "x86_avx512EXT_") # !subst(".", "_", mnemonic),
- [], [], traits, numResults>;
-
-// Defined by first result overload. May have to be extended for other
-// instructions in the future.
-class AVX512_IntrOverloadedOp<string mnemonic,
- list<Trait> traits = [],
- string extension = ""> :
- LLVM_IntrOpBase<X86Vector_Dialect, "avx512.intr." # mnemonic,
- !subst("EXT", extension, "x86_avx512EXT_") # !subst(".", "_", mnemonic),
- /*list<int> overloadedResults=*/[0],
- /*list<int> overloadedOperands=*/[],
- traits, /*numResults=*/1>;
-
//----------------------------------------------------------------------------//
// MaskCompressOp
//----------------------------------------------------------------------------//
@@ -91,21 +72,14 @@ def MaskCompressOp : AVX512_Op<"mask.compress", [Pure,
let assemblyFormat = "$k `,` $a (`,` $src^)? attr-dict"
" `:` type($dst) (`,` type($src)^)?";
let hasVerifier = 1;
-}
-def MaskCompressIntrOp : AVX512_IntrOverloadedOp<"mask.compress", [
- Pure,
- AllTypesMatch<["a", "src", "res"]>,
- TypesMatchWith<"`k` has the same number of bits as elements in `res`",
- "res", "k",
- "VectorType::get({::llvm::cast<VectorType>($_self).getShape()[0]}, "
- "IntegerType::get($_self.getContext(), 1))">]> {
- let arguments = (ins VectorOfLengthAndType<[16, 8],
- [F32, I32, F64, I64]>:$a,
- VectorOfLengthAndType<[16, 8],
- [F32, I32, F64, I64]>:$src,
- VectorOfLengthAndType<[16, 8],
- [I1]>:$k);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ // Overload is resolved later by intrisic call lowering.
+ return "llvm.x86.avx512.mask.compress";
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -142,26 +116,21 @@ def MaskRndScaleOp : AVX512_Op<"mask.rndscale", [Pure,
let results = (outs VectorOfLengthAndType<[16, 8], [F32, F64]>:$dst);
let assemblyFormat =
"$src `,` $k `,` $a `,` $imm `,` $rounding attr-dict `:` type($dst)";
-}
-def MaskRndScalePSIntrOp : AVX512_IntrOp<"mask.rndscale.ps.512", 1, [
- Pure,
- AllTypesMatch<["src", "a", "res"]>]> {
- let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
- I32:$k,
- VectorOfLengthAndType<[16], [F32]>:$a,
- I16:$imm,
- LLVM_Type:$rounding);
-}
-
-def MaskRndScalePDIntrOp : AVX512_IntrOp<"mask.rndscale.pd.512", 1, [
- Pure,
- AllTypesMatch<["src", "a", "res"]>]> {
- let arguments = (ins VectorOfLengthAndType<[8], [F64]>:$src,
- I32:$k,
- VectorOfLengthAndType<[8], [F64]>:$a,
- I8:$imm,
- LLVM_Type:$rounding);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ std::string intr = "llvm.x86.avx512.mask.rndscale";
+ VectorType vecType = getSrc().getType();
+ Type elemType = vecType.getElementType();
+ intr += ".";
+ intr += elemType.isF32() ? "ps" : "pd";
+ unsigned elemBitWidth = vecType.getElementTypeBitWidth();
+ unsigned opBitWidth = vecType.getShape()[0] * elemBitWidth;
+ intr += "." + std::to_string(opBitWidth);
+ return intr;
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -199,26 +168,21 @@ def MaskScaleFOp : AVX512_Op<"mask.scalef", [Pure,
// Fully specified by traits.
let assemblyFormat =
"$src `,` $a `,` $b `,` $k `,` $rounding attr-dict `:` type($dst)";
-}
-
-def MaskScaleFPSIntrOp : AVX512_IntrOp<"mask.scalef.ps.512", 1, [
- Pure,
- AllTypesMatch<["src", "a", "b", "res"]>]> {
- let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
- VectorOfLengthAndType<[16], [F32]>:$a,
- VectorOfLengthAndType<[16], [F32]>:$b,
- I16:$k,
- LLVM_Type:$rounding);
-}
-def MaskScaleFPDIntrOp : AVX512_IntrOp<"mask.scalef.pd.512", 1, [
- Pure,
- AllTypesMatch<["src", "a", "b", "res"]>]> {
- let arguments = (ins VectorOfLengthAndType<[8], [F64]>:$src,
- VectorOfLengthAndType<[8], [F64]>:$a,
- VectorOfLengthAndType<[8], [F64]>:$b,
- I8:$k,
- LLVM_Type:$rounding);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ std::string intr = "llvm.x86.avx512.mask.scalef";
+ VectorType vecType = getSrc().getType();
+ Type elemType = vecType.getElementType();
+ intr += ".";
+ intr += elemType.isF32() ? "ps" : "pd";
+ unsigned elemBitWidth = vecType.getElementTypeBitWidth();
+ unsigned opBitWidth = vecType.getShape()[0] * elemBitWidth;
+ intr += "." + std::to_string(opBitWidth);
+ return intr;
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -260,18 +224,21 @@ def Vp2IntersectOp : AVX512_Op<"vp2intersect", [Pure,
);
let assemblyFormat =
"$a `,` $b attr-dict `:` type($a)";
-}
-
-def Vp2IntersectDIntrOp : AVX512_IntrOp<"vp2intersect.d.512", 2, [
- Pure]> {
- let arguments = (ins VectorOfLengthAndType<[16], [I32]>:$a,
- VectorOfLengthAndType<[16], [I32]>:$b);
-}
-def Vp2IntersectQIntrOp : AVX512_IntrOp<"vp2intersect.q.512", 2, [
- Pure]> {
- let arguments = (ins VectorOfLengthAndType<[8], [I64]>:$a,
- VectorOfLengthAndType<[8], [I64]>:$b);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ std::string intr = "llvm.x86.avx512.vp2intersect";
+ VectorType vecType = getA().getType();
+ Type elemType = vecType.getElementType();
+ intr += ".";
+ intr += elemType.isInteger(32) ? "d" : "q";
+ unsigned elemBitWidth = vecType.getElementTypeBitWidth();
+ unsigned opBitWidth = vecType.getShape()[0] * elemBitWidth;
+ intr += "." + std::to_string(opBitWidth);
+ return intr;
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -299,7 +266,7 @@ def DotBF16Op : AVX512_Op<"dot", [Pure,
Example:
```mlir
- %0 = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
+ %dst = x86vector.avx512.dot %src, %a, %b : vector<32xbf16> -> vector<16xf32>
```
}];
let arguments = (ins VectorOfLengthAndType<[4, 8, 16], [F32]>:$src,
@@ -309,36 +276,18 @@ def DotBF16Op : AVX512_Op<"dot", [Pure,
let results = (outs VectorOfLengthAndType<[4, 8, 16], [F32]>:$dst);
let assemblyFormat =
"$src `,` $a `,` $b attr-dict `:` type($a) `->` type($src)";
-}
-
-def DotBF16Ps128IntrOp : AVX512_IntrOp<"dpbf16ps.128", 1, [Pure,
- AllTypesMatch<["a", "b"]>,
- AllTypesMatch<["src", "res"]>],
- /*extension=*/"bf16"> {
- let arguments = (ins VectorOfLengthAndType<[4], [F32]>:$src,
- VectorOfLengthAndType<[8], [BF16]>:$a,
- VectorOfLengthAndType<[8], [BF16]>:$b);
- let results = (outs VectorOfLengthAndType<[4], [F32]>:$res);
-}
-def DotBF16Ps256IntrOp : AVX512_IntrOp<"dpbf16ps.256", 1, [Pure,
- AllTypesMatch<["a", "b"]>,
- AllTypesMatch<["src", "res"]>],
- /*extension=*/"bf16"> {
- let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$src,
- VectorOfLengthAndType<[16], [BF16]>:$a,
- VectorOfLengthAndType<[16], [BF16]>:$b);
- let results = (outs VectorOfLengthAndType<[8], [F32]>:$res);
-}
-
-def DotBF16Ps512IntrOp : AVX512_IntrOp<"dpbf16ps.512", 1, [Pure,
- AllTypesMatch<["a", "b"]>,
- AllTypesMatch<["src", "res"]>],
- /*extension=*/"bf16"> {
- let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
- VectorOfLengthAndType<[32], [BF16]>:$a,
- VectorOfLengthAndType<[32], [BF16]>:$b);
- let results = (outs VectorOfLengthAndType<[16], [F32]>:$res);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ std::string intr = "llvm.x86.avx512bf16.dpbf16ps";
+ VectorType vecType = getSrc().getType();
+ unsigned elemBitWidth = vecType.getElementTypeBitWidth();
+ unsigned opBitWidth = vecType.getShape()[0] * elemBitWidth;
+ intr += "." + std::to_string(opBitWidth);
+ return intr;
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -367,18 +316,18 @@ def CvtPackedF32ToBF16Op : AVX512_Op<"cvt.packed.f32_to_bf16", [Pure,
let results = (outs VectorOfLengthAndType<[8, 16], [BF16]>:$dst);
let assemblyFormat =
"$a attr-dict `:` type($a) `->` type($dst)";
-}
-def CvtNeF32ToBF16Ps256IntrOp : AVX512_IntrOp<"cvtneps2bf16.256", 1, [Pure],
- /*extension=*/"bf16"> {
- let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$a);
- let results = (outs VectorOfLengthAndType<[8], [BF16]>:$res);
-}
-
-def CvtNeF32ToBF16Ps512IntrOp : AVX512_IntrOp<"cvtneps2bf16.512", 1, [Pure],
- /*extension=*/"bf16"> {
- let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$a);
- let results = (outs VectorOfLengthAndType<[16], [BF16]>:$res);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ std::string intr = "llvm.x86.avx512bf16.cvtneps2bf16";
+ VectorType vecType = getA().getType();
+ unsigned elemBitWidth = vecType.getElementTypeBitWidth();
+ unsigned opBitWidth = vecType.getShape()[0] * elemBitWidth;
+ intr += "." + std::to_string(opBitWidth);
+ return intr;
+ }
+ }];
}
//===----------------------------------------------------------------------===//
@@ -395,12 +344,6 @@ class AVX_Op<string mnemonic, list<Trait> traits = []> :
class AVX_LowOp<string mnemonic, list<Trait> traits = []> :
Op<X86Vector_Dialect, "avx.intr." # mnemonic, traits> {}
-// Intrinsic operation used during lowering to LLVM IR.
-class AVX_IntrOp<string mnemonic, int numResults, list<Trait> traits = []> :
- LLVM_IntrOpBase<X86Vector_Dialect, "avx.intr." # mnemonic,
- "x86_avx_" # !subst(".", "_", mnemonic),
- [], [], traits, numResults>;
-
//----------------------------------------------------------------------------//
// AVX Rsqrt
//----------------------------------------------------------------------------//
@@ -410,11 +353,13 @@ def RsqrtOp : AVX_Op<"rsqrt", [Pure, SameOperandsAndResultType]> {
let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$a);
let results = (outs VectorOfLengthAndType<[8], [F32]>:$b);
let assemblyFormat = "$a attr-dict `:` type($a)";
-}
-def RsqrtIntrOp : AVX_IntrOp<"rsqrt.ps.256", 1, [Pure,
- SameOperandsAndResultType]> {
- let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$a);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ return "llvm.x86.avx.rsqrt.ps.256";
+ }
+ }];
}
//----------------------------------------------------------------------------//
@@ -443,13 +388,13 @@ def DotOp : AVX_LowOp<"dot", [Pure, SameOperandsAndResultType]> {
VectorOfLengthAndType<[8], [F32]>:$b);
let results = (outs VectorOfLengthAndType<[8], [F32]>:$res);
let assemblyFormat = "$a `,` $b attr-dict `:` type($res)";
-}
-def DotIntrOp : AVX_IntrOp<"dp.ps.256", 1, [Pure,
- AllTypesMatch<["a", "b", "res"]>]> {
- let arguments = (ins VectorOfLengthAndType<[8], [F32]>:$a,
- VectorOfLengthAndType<[8], [F32]>:$b, I8:$c);
- let results = (outs VectorOfLengthAndType<[8], [F32]>:$res);
+ let extraClassDeclaration = [{
+ /// Return LLVM intrinsic function name matching op variant.
+ std::string getIntrinsicName() {
+ return "llvm.x86.avx.dp.ps.256";
+ }
+ }];
}
#endif // X86VECTOR_OPS
diff --git a/mlir/include/mlir/Target/LLVMIR/Dialect/All.h b/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
index de9d5872cc454..e043ff2f6825c 100644
--- a/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
+++ b/mlir/include/mlir/Target/LLVMIR/Dialect/All.h
@@ -29,7 +29,6 @@
#include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.h"
-#include "mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h"
namespace mlir {
class DialectRegistry;
@@ -50,7 +49,6 @@ static inline void registerAllToLLVMIRTranslations(DialectRegistry ®istry) {
registerROCDLDialectTranslation(registry);
registerSPIRVDialectTranslation(registry);
registerVCIXDialectTranslation(registry);
- registerX86VectorDialectTranslation(registry);
// Extension required for translating GPU offloading Ops.
gpu::registerOffloadingLLVMTranslationInterfaceExternalModels(registry);
diff --git a/mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h b/mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h
deleted file mode 100644
index a215bcf625ae9..0000000000000
--- a/mlir/include/mlir/Target/LLVMIR/Dialect/X86Vector/X86VectorToLLVMIRTranslation.h
+++ /dev/null
@@ -1,32 +0,0 @@
-//===- X86VectorToLLVMIRTranslation.h - X86Vector to LLVM IR ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This provides registration calls for X86Vector dialect to LLVM IR
-// translation.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_TARGET_LLVMIR_DIALECT_X86VECTOR_X86VECTORTOLLVMIRTRANSLATION_H
-#define MLIR_TARGET_LLVMIR_DIALECT_X86VECTOR_X86VECTORTOLLVMIRTRANSLATION_H
-
-namespace mlir {
-
-class DialectRegistry;
-class MLIRContext;
-
-/// Register the X86Vector dialect and the translation from it to the LLVM IR
-/// in the given registry;
-void registerX86VectorDialectTranslation(DialectRegistry ®istry);
-
-/// Register the X86Vector dialect and the translation from it in the registry
-/// associated with the given context.
-void registerX86VectorDialectTranslation(MLIRContext &context);
-
-} // namespace mlir
-
-#endif // MLIR_TARGET_LLVMIR_DIALECT_X86VECTOR_X86VECTORTOLLVMIRTRANSLATION_H
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/CMakeLists.txt b/mlir/lib/Dialect/X86Vector/Transforms/CMakeLists.txt
index f1fb64ffa3f4d..c51266afe9e8f 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/X86Vector/Transforms/CMakeLists.txt
@@ -2,9 +2,6 @@ add_mlir_dialect_library(MLIRX86VectorTransforms
AVXTranspose.cpp
LegalizeForLLVMExport.cpp
- DEPENDS
- MLIRX86VectorConversionsIncGen
-
LINK_LIBS PUBLIC
MLIRArithDialect
MLIRX86VectorDialect
diff --git a/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
index f1fbb39b97fc4..350c9b59e5d23 100644
--- a/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/X86Vector/Transforms/LegalizeForLLVMExport.cpp
@@ -10,7 +10,6 @@
#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
#include "mlir/Conversion/LLVMCommon/Pattern.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/X86Vector/X86VectorDialect.h"
#include "mlir/IR/BuiltinOps.h"
@@ -19,47 +18,53 @@
using namespace mlir;
using namespace mlir::x86vector;
-/// Extracts the "main" vector element type from the given X86Vector operation.
-template <typename OpTy>
-static Type getSrcVectorElementType(OpTy op) {
- return cast<VectorType>(op.getSrc().getType()).getElementType();
-}
-template <>
-Type getSrcVectorElementType(Vp2IntersectOp op) {
- return cast<VectorType>(op.getA().getType()).getElementType();
-}
-
namespace {
-/// Base conversion for AVX512 ops that can be lowered to one of the two
-/// intrinsics based on the bitwidth of their "main" vector element type. This
-/// relies on the to-LLVM-dialect conversion helpers to correctly pack the
-/// results of multi-result intrinsic ops.
-template <typename OpTy, typename Intr32OpTy, typename Intr64OpTy>
-struct LowerToIntrinsic : public OpConversionPattern<OpTy> {
- explicit LowerToIntrinsic(const LLVMTypeConverter &converter)
- : OpConversionPattern<OpTy>(converter, &converter.getContext()) {}
-
- const LLVMTypeConverter &getTypeConverter() const {
- return *static_cast<const LLVMTypeConverter *>(
- OpConversionPattern<OpTy>::getTypeConverter());
+// Replaces an operation with a call to an LLVM intrinsic.
+LogicalResult intrinsicRewrite(Operation *op, StringAttr intrinsic,
+ ValueRange operands,
+ const LLVMTypeConverter &typeConverter,
+ ConversionPatternRewriter &rewriter) {
+ auto loc = op->getLoc();
+
+ unsigned numResults = op->getNumResults();
+ Type resType;
+ if (numResults != 0)
+ resType = typeConverter.packOperationResults(op->getResultTypes());
+
+ auto callIntrOp =
+ rewriter.create<LLVM::CallIntrinsicOp>(loc, resType, intrinsic, operands);
+ // Propagate attributes.
+ callIntrOp->setAttrs(op->getAttrDictionary());
+
+ if (numResults <= 1) {
+ // Directly replace the original op.
+ rewriter.replaceOp(op, callIntrOp);
+ } else {
+ // Extract individual results from packed structure and use them as
+ // replacements.
+ SmallVector<Value, 4> results;
+ results.reserve(numResults);
+ Value intrRes = callIntrOp.getResults();
+ for (unsigned i = 0; i < numResults; ++i) {
+ results.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, intrRes, i));
+ }
+ rewriter.replaceOp(op, results);
}
+ return success();
+}
+
+template <typename OpTy>
+struct CallIntrinsic : public ConvertOpToLLVMPattern<OpTy> {
+ using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
+
LogicalResult
matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- Type elementType = getSrcVectorElementType<OpTy>(op);
- unsigned bitwidth = elementType.getIntOrFloatBitWidth();
- if (bitwidth == 32)
- return LLVM::detail::oneToOneRewrite(
- ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/133692
More information about the llvm-commits
mailing list