[clang] Implement select/selectsh builtins in CIR (PR #172299)
Priyanshu Kumar via cfe-commits
cfe-commits at lists.llvm.org
Mon Dec 15 10:14:01 PST 2025
https://github.com/Priyanshu3820 updated https://github.com/llvm/llvm-project/pull/172299
>From 2fc787b56e25b558e4740f88f9406d806705087f Mon Sep 17 00:00:00 2001
From: Priyanshu3820 <10b.priyanshu at gmail.com>
Date: Mon, 15 Dec 2025 19:27:20 +0530
Subject: [PATCH 1/2] Emit CIR op for select/selectsh X86 builtins
---
clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp | 20 ++
.../X86/avx512-select-builtins.c | 292 ++++++++++++++++++
2 files changed, 312 insertions(+)
create mode 100644 clang/test/CIR/CodeGenBuiltins/X86/avx512-select-builtins.c
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
index fb17e31bf36d6..b3182fc83776e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
@@ -370,6 +370,22 @@ static mlir::Value emitX86vpcom(CIRGenBuilderTy &builder, mlir::Location loc,
return builder.createVecCompare(loc, pred, op0, op1);
}
+static mlir::Value emitX86Select(CIRGenBuilderTy &builder, mlir::Location loc,
+ mlir::Value mask, mlir::Value Op0,
+ mlir::Value Op1) {
+ return builder.create<cir::VecTernaryOp>(loc, Op0.getType(), mask, Op0, Op1);
+}
+
+static mlir::Value emitX86ScalarSelect(CIRGenBuilderTy &builder,
+ mlir::Location loc, mlir::Value mask,
+ mlir::Value Op0, mlir::Value Op1) {
+
+ mlir::Value zero = builder.getZero(mask.getType(), loc);
+ mlir::Value cond = builder.createCompare(loc, CmpOpKind::ne, mask, zero);
+
+ return builder.createSelect(loc, Op0.getType(), cond, Op0, Op1);
+}
+
mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
const CallExpr *expr) {
if (builtinID == Builtin::BI__builtin_cpu_is) {
@@ -1186,10 +1202,14 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
case X86::BI__builtin_ia32_selectpd_128:
case X86::BI__builtin_ia32_selectpd_256:
case X86::BI__builtin_ia32_selectpd_512:
+ return emitX86Select(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
+ ops[2]);
case X86::BI__builtin_ia32_selectsh_128:
case X86::BI__builtin_ia32_selectsbf_128:
case X86::BI__builtin_ia32_selectss_128:
case X86::BI__builtin_ia32_selectsd_128:
+ return emitX86ScalarSelect(builder, getLoc(expr->getExprLoc()), ops[0],
+ ops[1], ops[2]);
case X86::BI__builtin_ia32_cmpb128_mask:
case X86::BI__builtin_ia32_cmpb256_mask:
case X86::BI__builtin_ia32_cmpb512_mask:
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx512-select-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512-select-builtins.c
new file mode 100644
index 0000000000000..68edcbee8fcb6
--- /dev/null
+++ b/clang/test/CIR/CodeGenBuiltins/X86/avx512-select-builtins.c
@@ -0,0 +1,292 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512bw -target-feature +avx512vl -fclangir -emit-cir %s -o - | FileCheck %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512bw -target-feature +avx512vl -emit-llvm %s -o - | FileCheck %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512bw -target-feature +avx512vl -emit-llvm %s -o - | FileCheck %s --check-prefix=OGCG
+
+// REQUIRES: avx512bw
+// REQUIRES: avx512vl
+
+#include <immintrin.h>
+
+// CIR-LABEL: test_selectb_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectb_128
+// LLVM: select <16 x i8>
+// OGCG-LABEL: test_selectb_128
+// OGCG: select <16 x i8>
+__m128i test_selectb_128(__mmask16 k, __m128i a, __m128i b) {
+ return _mm_selectb_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectb_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectb_256
+// LLVM: select <32 x i8>
+// OGCG-LABEL: test_selectb_256
+// OGCG: select <32 x i8>
+__m256i test_selectb_256(__mmask32 k, __m256i a, __m256i b) {
+ return _mm256_selectb_epi8(k, a, b);
+}
+
+// CIR-LABEL: test_selectb_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectb_512
+// LLVM: select <64 x i8>
+// OGCG-LABEL: test_selectb_512
+// OGCG: select <64 x i8>
+__m512i test_selectb_512(__mmask64 k, __m512i a, __m512i b) {
+ return _mm512_selectb_epi8(k, a, b);
+}
+
+// CIR-LABEL: test_selectw_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectw_128
+// LLVM: select <8 x i16>
+// OGCG-LABEL: test_selectw_128
+// OGCG: select <8 x i16>
+__m128i test_selectw_128(__mmask8 k, __m128i a, __m128i b) {
+ return _mm_selectw_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectw_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectw_256
+// LLVM: select <16 x i16>
+// OGCG-LABEL: test_selectw_256
+// OGCG: select <16 x i16>
+__m256i test_selectw_256(__mmask16 k, __m256i a, __m256i b) {
+ return _mm256_selectw_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectw_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectw_512
+// LLVM: select <32 x i16>
+// OGCG-LABEL: test_selectw_512
+// OGCG: select <32 x i16>
+__m512i test_selectw_512(__mmask32 k, __m512i a, __m512i b) {
+ return _mm512_selectw_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectd_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectd_128
+// LLVM: select <4 x i32>
+// OGCG-LABEL: test_selectd_128
+// OGCG: select <4 x i32>
+__m128i test_selectd_128(__mmask4 k, __m128i a, __m128i b) {
+ return _mm_selectd_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectd_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectd_256
+// LLVM: select <8 x i32>
+// OGCG-LABEL: test_selectd_256
+// OGCG: select <8 x i32>
+__m256i test_selectd_256(__mmask8 k, __m256i a, __m256i b) {
+ return _mm256_selectd_epi32(k, a, b);
+}
+
+// CIR-LABEL: test_selectd_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectd_512
+// LLVM: select <16 x i32>
+// OGCG-LABEL: test_selectd_512
+// OGCG: select <16 x i32>
+__m512i test_selectd_512(__mmask16 k, __m512i a, __m512i b) {
+ return _mm512_selectd_epi32(k, a, b);
+}
+
+// CIR-LABEL: test_selectq_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectq_128
+// LLVM: select <2 x i64>
+// OGCG-LABEL: test_selectq_128
+// OGCG: select <2 x i64>
+__m128i test_selectq_128(__mmask2 k, __m128i a, __m128i b) {
+ return _mm_selectq_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectq_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectq_256
+// LLVM: select <4 x i64>
+// OGCG-LABEL: test_selectq_256
+// OGCG: select <4 x i64>
+__m256i test_selectq_256(__mmask4 k, __m256i a, __m256i b) {
+ return _mm256_selectq_epi64(k, a, b);
+}
+
+// CIR-LABEL: test_selectq_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectq_512
+// LLVM: select <8 x i64>
+// OGCG-LABEL: test_selectq_512
+// OGCG: select <8 x i64>
+__m512i test_selectq_512(__mmask8 k, __m512i a, __m512i b) {
+ return _mm512_selectq_epi64(k, a, b);
+}
+
+// CIR-LABEL: test_selectph_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectph_128
+// LLVM: select
+// OGCG-LABEL: test_selectph_128
+// OGCG: select
+__m128i test_selectph_128(__mmask8 k, __m128i a, __m128i b) {
+ return _mm_selectph_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectph_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectph_256
+// LLVM: select
+// OGCG-LABEL: test_selectph_256
+// OGCG: select
+__m256i test_selectph_256(__mmask16 k, __m256i a, __m256i b) {
+ return _mm256_selectph_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectph_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectph_512
+// LLVM: select
+// OGCG-LABEL: test_selectph_512
+// OGCG: select
+__m512i test_selectph_512(__mmask32 k, __m512i a, __m512i b) {
+ return _mm512_selectph_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectpbf_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpbf_128
+// LLVM: select
+// OGCG-LABEL: test_selectpbf_128
+// OGCG: select
+__m128i test_selectpbf_128(__mmask8 k, __m128i a, __m128i b) {
+ return _mm_selectpbf_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectpbf_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpbf_256
+// LLVM: select
+// OGCG-LABEL: test_selectpbf_256
+// OGCG: select
+__m256i test_selectpbf_256(__mmask16 k, __m256i a, __m256i b) {
+ return _mm256_selectpbf_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectpbf_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpbf_512
+// LLVM: select
+// OGCG-LABEL: test_selectpbf_512
+// OGCG: select
+__m512i test_selectpbf_512(__mmask32 k, __m512i a, __m512i b) {
+ return _mm512_selectpbf_epi16(k, a, b);
+}
+
+// CIR-LABEL: test_selectps_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectps_128
+// LLVM: select
+// OGCG-LABEL: test_selectps_128
+// OGCG: select
+__m128 test_selectps_128(__mmask8 k, __m128 a, __m128 b) {
+ return _mm_selectps_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectps_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectps_256
+// LLVM: select
+// OGCG-LABEL: test_selectps_256
+// OGCG: select
+__m256 test_selectps_256(__mmask8 k, __m256 a, __m256 b) {
+ return _mm256_selectps(k, a, b);
+}
+
+// CIR-LABEL: test_selectps_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectps_512
+// LLVM: select
+// OGCG-LABEL: test_selectps_512
+// OGCG: select
+__m512 test_selectps_512(__mmask16 k, __m512 a, __m512 b) {
+ return _mm512_selectps(k, a, b);
+}
+
+// CIR-LABEL: test_selectpd_128
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpd_128
+// LLVM: select
+// OGCG-LABEL: test_selectpd_128
+// OGCG: select
+__m128d test_selectpd_128(__mmask8 k, __m128d a, __m128d b) {
+ return _mm_selectpd_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectpd_256
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpd_256
+// LLVM: select
+// OGCG-LABEL: test_selectpd_256
+// OGCG: select
+__m256d test_selectpd_256(__mmask8 k, __m256d a, __m256d b) {
+ return _mm256_selectpd(k, a, b);
+}
+
+// CIR-LABEL: test_selectpd_512
+// CIR: cir.vec.ternary
+// LLVM-LABEL: test_selectpd_512
+// LLVM: select
+// OGCG-LABEL: test_selectpd_512
+// OGCG: select
+__m512d test_selectpd_512(__mmask8 k, __m512d a, __m512d b) {
+ return _mm512_selectpd(k, a, b);
+}
+
+// CIR-LABEL: test_selectsh_128
+// CIR: cir.cmp {{.*}} ne
+// CIR: cir.select
+// LLVM-LABEL: test_selectsh_128
+// LLVM: select
+// OGCG-LABEL: test_selectsh_128
+// OGCG: select
+__m128i test_selectsh_128(unsigned short k, __m128i a, __m128i b) {
+ return _mm_selectsh_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectsbf_128
+// CIR: cir.cmp {{.*}} ne
+// CIR: cir.select
+// LLVM-LABEL: test_selectsbf_128
+// LLVM: select
+// OGCG-LABEL: test_selectsbf_128
+// OGCG: select
+__m128i test_selectsbf_128(unsigned short k, __m128i a, __m128i b) {
+ return _mm_selectsbf_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectss_128
+// CIR: cir.cmp {{.*}} ne
+// CIR: cir.select
+// LLVM-LABEL: test_selectss_128
+// LLVM: select
+// OGCG-LABEL: test_selectss_128
+// OGCG: select
+__m128 test_selectss_128(unsigned short k, __m128 a, __m128 b) {
+ return _mm_selectss_128(k, a, b);
+}
+
+// CIR-LABEL: test_selectsd_128
+// CIR: cir.cmp {{.*}} ne
+// CIR: cir.select
+// LLVM-LABEL: test_selectsd_128
+// LLVM: select
+// OGCG-LABEL: test_selectsd_128
+// OGCG: select
+__m128d test_selectsd_128(unsigned short k, __m128d a, __m128d b) {
+ return _mm_selectsd_128(k, a, b);
+}
>From ca1ce501958d01242e25033ec9513bb06c805f34 Mon Sep 17 00:00:00 2001
From: Priyanshu Kumar <10b.priyanshu at gmail.com>
Date: Mon, 15 Dec 2025 18:13:18 +0000
Subject: [PATCH 2/2] Fix formatting
---
clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp | 2569 ++++++++++----------
1 file changed, 1287 insertions(+), 1282 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
index eb8773be5b87b..480f1b592ad9b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
@@ -435,1357 +435,1362 @@ static mlir::Value emitX86ScalarSelect(CIRGenBuilderTy &builder,
mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
const CallExpr *expr) {
-std::optional<mlir::Value>
-CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID, const CallExpr *expr) {
- if (builtinID == Builtin::BI__builtin_cpu_is) {
- cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_is");
- return mlir::Value{};
- }
- if (builtinID == Builtin::BI__builtin_cpu_supports) {
- cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_supports");
- return mlir::Value{};
- }
- if (builtinID == Builtin::BI__builtin_cpu_init) {
- cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_init");
- return mlir::Value{};
- }
+ std::optional<mlir::Value> CIRGenFunction::emitX86BuiltinExpr(
+ unsigned builtinID, const CallExpr *expr) {
+ if (builtinID == Builtin::BI__builtin_cpu_is) {
+ cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_is");
+ return mlir::Value{};
+ }
+ if (builtinID == Builtin::BI__builtin_cpu_supports) {
+ cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_supports");
+ return mlir::Value{};
+ }
+ if (builtinID == Builtin::BI__builtin_cpu_init) {
+ cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_init");
+ return mlir::Value{};
+ }
- // Handle MSVC intrinsics before argument evaluation to prevent double
- // evaluation.
- assert(!cir::MissingFeatures::msvcBuiltins());
+ // Handle MSVC intrinsics before argument evaluation to prevent double
+ // evaluation.
+ assert(!cir::MissingFeatures::msvcBuiltins());
- // Find out if any arguments are required to be integer constant expressions.
- assert(!cir::MissingFeatures::handleBuiltinICEArguments());
+ // Find out if any arguments are required to be integer constant
+ // expressions.
+ assert(!cir::MissingFeatures::handleBuiltinICEArguments());
- // The operands of the builtin call
- llvm::SmallVector<mlir::Value> ops;
+ // The operands of the builtin call
+ llvm::SmallVector<mlir::Value> ops;
- // `ICEArguments` is a bitmap indicating whether the argument at the i-th bit
- // is required to be a constant integer expression.
- unsigned iceArguments = 0;
- ASTContext::GetBuiltinTypeError error;
- getContext().GetBuiltinType(builtinID, error, &iceArguments);
- assert(error == ASTContext::GE_None && "Error while getting builtin type.");
+ // `ICEArguments` is a bitmap indicating whether the argument at the i-th
+ // bit is required to be a constant integer expression.
+ unsigned iceArguments = 0;
+ ASTContext::GetBuiltinTypeError error;
+ getContext().GetBuiltinType(builtinID, error, &iceArguments);
+ assert(error == ASTContext::GE_None && "Error while getting builtin type.");
- for (auto [idx, arg] : llvm::enumerate(expr->arguments()))
- ops.push_back(emitScalarOrConstFoldImmArg(iceArguments, idx, arg));
+ for (auto [idx, arg] : llvm::enumerate(expr->arguments()))
+ ops.push_back(emitScalarOrConstFoldImmArg(iceArguments, idx, arg));
- CIRGenBuilderTy &builder = getBuilder();
- mlir::Type voidTy = builder.getVoidTy();
+ CIRGenBuilderTy &builder = getBuilder();
+ mlir::Type voidTy = builder.getVoidTy();
- switch (builtinID) {
- default:
- return std::nullopt;
- case X86::BI_mm_clflush:
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.sse2.clflush", voidTy, ops[0]);
- case X86::BI_mm_lfence:
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.sse2.lfence", voidTy);
- case X86::BI_mm_pause:
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.sse2.pause", voidTy);
- case X86::BI_mm_mfence:
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.sse2.mfence", voidTy);
- case X86::BI_mm_sfence:
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.sse.sfence", voidTy);
- case X86::BI_mm_prefetch:
- case X86::BI__rdtsc:
- case X86::BI__builtin_ia32_rdtscp: {
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- }
- case X86::BI__builtin_ia32_lzcnt_u16:
- case X86::BI__builtin_ia32_lzcnt_u32:
- case X86::BI__builtin_ia32_lzcnt_u64: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- mlir::Value isZeroPoison = builder.getFalse(loc);
- return emitIntrinsicCallOp(builder, loc, "ctlz", ops[0].getType(),
- mlir::ValueRange{ops[0], isZeroPoison});
- }
- case X86::BI__builtin_ia32_tzcnt_u16:
- case X86::BI__builtin_ia32_tzcnt_u32:
- case X86::BI__builtin_ia32_tzcnt_u64: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- mlir::Value isZeroPoison = builder.getFalse(loc);
- return emitIntrinsicCallOp(builder, loc, "cttz", ops[0].getType(),
- mlir::ValueRange{ops[0], isZeroPoison});
- }
- case X86::BI__builtin_ia32_undef128:
- case X86::BI__builtin_ia32_undef256:
- case X86::BI__builtin_ia32_undef512:
- // The x86 definition of "undef" is not the same as the LLVM definition
- // (PR32176). We leave optimizing away an unnecessary zero constant to the
- // IR optimizer and backend.
- // TODO: If we had a "freeze" IR instruction to generate a fixed undef
- // value, we should use that here instead of a zero.
- return builder.getNullValue(convertType(expr->getType()),
- getLoc(expr->getExprLoc()));
- case X86::BI__builtin_ia32_vec_ext_v4hi:
- case X86::BI__builtin_ia32_vec_ext_v16qi:
- case X86::BI__builtin_ia32_vec_ext_v8hi:
- case X86::BI__builtin_ia32_vec_ext_v4si:
- case X86::BI__builtin_ia32_vec_ext_v4sf:
- case X86::BI__builtin_ia32_vec_ext_v2di:
- case X86::BI__builtin_ia32_vec_ext_v32qi:
- case X86::BI__builtin_ia32_vec_ext_v16hi:
- case X86::BI__builtin_ia32_vec_ext_v8si:
- case X86::BI__builtin_ia32_vec_ext_v4di: {
- unsigned numElts = cast<cir::VectorType>(ops[0].getType()).getSize();
-
- uint64_t index = getZExtIntValueFromConstOp(ops[1]);
- index &= numElts - 1;
-
- cir::ConstantOp indexVal =
- builder.getUInt64(index, getLoc(expr->getExprLoc()));
-
- // These builtins exist so we can ensure the index is an ICE and in range.
- // Otherwise we could just do this in the header file.
- return cir::VecExtractOp::create(builder, getLoc(expr->getExprLoc()),
- ops[0], indexVal);
- }
- case X86::BI__builtin_ia32_vec_set_v4hi:
- case X86::BI__builtin_ia32_vec_set_v16qi:
- case X86::BI__builtin_ia32_vec_set_v8hi:
- case X86::BI__builtin_ia32_vec_set_v4si:
- case X86::BI__builtin_ia32_vec_set_v2di:
- case X86::BI__builtin_ia32_vec_set_v32qi:
- case X86::BI__builtin_ia32_vec_set_v16hi:
- case X86::BI__builtin_ia32_vec_set_v8si:
- case X86::BI__builtin_ia32_vec_set_v4di: {
- return emitVecInsert(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
- ops[2]);
- }
- case X86::BI__builtin_ia32_kunpckhi:
- return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kunpackb", ops);
- case X86::BI__builtin_ia32_kunpcksi:
- return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kunpackw", ops);
- case X86::BI__builtin_ia32_kunpckdi:
- return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kunpackd", ops);
- case X86::BI_mm_setcsr:
- case X86::BI__builtin_ia32_ldmxcsr: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- Address tmp = createMemTemp(expr->getArg(0)->getType(), loc);
- builder.createStore(loc, ops[0], tmp);
- return emitIntrinsicCallOp(builder, loc, "x86.sse.ldmxcsr",
- builder.getVoidTy(), tmp.getPointer());
- }
- case X86::BI_mm_getcsr:
- case X86::BI__builtin_ia32_stmxcsr: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- Address tmp = createMemTemp(expr->getType(), loc);
- emitIntrinsicCallOp(builder, loc, "x86.sse.stmxcsr", builder.getVoidTy(),
- tmp.getPointer());
- return builder.createLoad(loc, tmp);
- }
- case X86::BI__builtin_ia32_xsave:
- case X86::BI__builtin_ia32_xsave64:
- case X86::BI__builtin_ia32_xrstor:
- case X86::BI__builtin_ia32_xrstor64:
- case X86::BI__builtin_ia32_xsaveopt:
- case X86::BI__builtin_ia32_xsaveopt64:
- case X86::BI__builtin_ia32_xrstors:
- case X86::BI__builtin_ia32_xrstors64:
- case X86::BI__builtin_ia32_xsavec:
- case X86::BI__builtin_ia32_xsavec64:
- case X86::BI__builtin_ia32_xsaves:
- case X86::BI__builtin_ia32_xsaves64:
- case X86::BI__builtin_ia32_xsetbv:
- case X86::BI_xsetbv: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- StringRef intrinsicName;
switch (builtinID) {
default:
- llvm_unreachable("Unexpected builtin");
+ return std::nullopt;
+ case X86::BI_mm_clflush:
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.sse2.clflush", voidTy, ops[0]);
+ case X86::BI_mm_lfence:
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.sse2.lfence", voidTy);
+ case X86::BI_mm_pause:
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.sse2.pause", voidTy);
+ case X86::BI_mm_mfence:
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.sse2.mfence", voidTy);
+ case X86::BI_mm_sfence:
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.sse.sfence", voidTy);
+ case X86::BI_mm_prefetch:
+ case X86::BI__rdtsc:
+ case X86::BI__builtin_ia32_rdtscp: {
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ }
+ case X86::BI__builtin_ia32_lzcnt_u16:
+ case X86::BI__builtin_ia32_lzcnt_u32:
+ case X86::BI__builtin_ia32_lzcnt_u64: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ mlir::Value isZeroPoison = builder.getFalse(loc);
+ return emitIntrinsicCallOp(builder, loc, "ctlz", ops[0].getType(),
+ mlir::ValueRange{ops[0], isZeroPoison});
+ }
+ case X86::BI__builtin_ia32_tzcnt_u16:
+ case X86::BI__builtin_ia32_tzcnt_u32:
+ case X86::BI__builtin_ia32_tzcnt_u64: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ mlir::Value isZeroPoison = builder.getFalse(loc);
+ return emitIntrinsicCallOp(builder, loc, "cttz", ops[0].getType(),
+ mlir::ValueRange{ops[0], isZeroPoison});
+ }
+ case X86::BI__builtin_ia32_undef128:
+ case X86::BI__builtin_ia32_undef256:
+ case X86::BI__builtin_ia32_undef512:
+ // The x86 definition of "undef" is not the same as the LLVM definition
+ // (PR32176). We leave optimizing away an unnecessary zero constant to the
+ // IR optimizer and backend.
+ // TODO: If we had a "freeze" IR instruction to generate a fixed undef
+ // value, we should use that here instead of a zero.
+ return builder.getNullValue(convertType(expr->getType()),
+ getLoc(expr->getExprLoc()));
+ case X86::BI__builtin_ia32_vec_ext_v4hi:
+ case X86::BI__builtin_ia32_vec_ext_v16qi:
+ case X86::BI__builtin_ia32_vec_ext_v8hi:
+ case X86::BI__builtin_ia32_vec_ext_v4si:
+ case X86::BI__builtin_ia32_vec_ext_v4sf:
+ case X86::BI__builtin_ia32_vec_ext_v2di:
+ case X86::BI__builtin_ia32_vec_ext_v32qi:
+ case X86::BI__builtin_ia32_vec_ext_v16hi:
+ case X86::BI__builtin_ia32_vec_ext_v8si:
+ case X86::BI__builtin_ia32_vec_ext_v4di: {
+ unsigned numElts = cast<cir::VectorType>(ops[0].getType()).getSize();
+
+ uint64_t index = getZExtIntValueFromConstOp(ops[1]);
+ index &= numElts - 1;
+
+ cir::ConstantOp indexVal =
+ builder.getUInt64(index, getLoc(expr->getExprLoc()));
+
+ // These builtins exist so we can ensure the index is an ICE and in range.
+ // Otherwise we could just do this in the header file.
+ return cir::VecExtractOp::create(builder, getLoc(expr->getExprLoc()),
+ ops[0], indexVal);
+ }
+ case X86::BI__builtin_ia32_vec_set_v4hi:
+ case X86::BI__builtin_ia32_vec_set_v16qi:
+ case X86::BI__builtin_ia32_vec_set_v8hi:
+ case X86::BI__builtin_ia32_vec_set_v4si:
+ case X86::BI__builtin_ia32_vec_set_v2di:
+ case X86::BI__builtin_ia32_vec_set_v32qi:
+ case X86::BI__builtin_ia32_vec_set_v16hi:
+ case X86::BI__builtin_ia32_vec_set_v8si:
+ case X86::BI__builtin_ia32_vec_set_v4di: {
+ return emitVecInsert(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
+ ops[2]);
+ }
+ case X86::BI__builtin_ia32_kunpckhi:
+ return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kunpackb", ops);
+ case X86::BI__builtin_ia32_kunpcksi:
+ return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kunpackw", ops);
+ case X86::BI__builtin_ia32_kunpckdi:
+ return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kunpackd", ops);
+ case X86::BI_mm_setcsr:
+ case X86::BI__builtin_ia32_ldmxcsr: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ Address tmp = createMemTemp(expr->getArg(0)->getType(), loc);
+ builder.createStore(loc, ops[0], tmp);
+ return emitIntrinsicCallOp(builder, loc, "x86.sse.ldmxcsr",
+ builder.getVoidTy(), tmp.getPointer());
+ }
+ case X86::BI_mm_getcsr:
+ case X86::BI__builtin_ia32_stmxcsr: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ Address tmp = createMemTemp(expr->getType(), loc);
+ emitIntrinsicCallOp(builder, loc, "x86.sse.stmxcsr", builder.getVoidTy(),
+ tmp.getPointer());
+ return builder.createLoad(loc, tmp);
+ }
case X86::BI__builtin_ia32_xsave:
- intrinsicName = "x86.xsave";
- break;
case X86::BI__builtin_ia32_xsave64:
- intrinsicName = "x86.xsave64";
- break;
case X86::BI__builtin_ia32_xrstor:
- intrinsicName = "x86.xrstor";
- break;
case X86::BI__builtin_ia32_xrstor64:
- intrinsicName = "x86.xrstor64";
- break;
case X86::BI__builtin_ia32_xsaveopt:
- intrinsicName = "x86.xsaveopt";
- break;
case X86::BI__builtin_ia32_xsaveopt64:
- intrinsicName = "x86.xsaveopt64";
- break;
case X86::BI__builtin_ia32_xrstors:
- intrinsicName = "x86.xrstors";
- break;
case X86::BI__builtin_ia32_xrstors64:
- intrinsicName = "x86.xrstors64";
- break;
case X86::BI__builtin_ia32_xsavec:
- intrinsicName = "x86.xsavec";
- break;
case X86::BI__builtin_ia32_xsavec64:
- intrinsicName = "x86.xsavec64";
- break;
case X86::BI__builtin_ia32_xsaves:
- intrinsicName = "x86.xsaves";
- break;
case X86::BI__builtin_ia32_xsaves64:
- intrinsicName = "x86.xsaves64";
- break;
case X86::BI__builtin_ia32_xsetbv:
- case X86::BI_xsetbv:
- intrinsicName = "x86.xsetbv";
- break;
+ case X86::BI_xsetbv: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ StringRef intrinsicName;
+ switch (builtinID) {
+ default:
+ llvm_unreachable("Unexpected builtin");
+ case X86::BI__builtin_ia32_xsave:
+ intrinsicName = "x86.xsave";
+ break;
+ case X86::BI__builtin_ia32_xsave64:
+ intrinsicName = "x86.xsave64";
+ break;
+ case X86::BI__builtin_ia32_xrstor:
+ intrinsicName = "x86.xrstor";
+ break;
+ case X86::BI__builtin_ia32_xrstor64:
+ intrinsicName = "x86.xrstor64";
+ break;
+ case X86::BI__builtin_ia32_xsaveopt:
+ intrinsicName = "x86.xsaveopt";
+ break;
+ case X86::BI__builtin_ia32_xsaveopt64:
+ intrinsicName = "x86.xsaveopt64";
+ break;
+ case X86::BI__builtin_ia32_xrstors:
+ intrinsicName = "x86.xrstors";
+ break;
+ case X86::BI__builtin_ia32_xrstors64:
+ intrinsicName = "x86.xrstors64";
+ break;
+ case X86::BI__builtin_ia32_xsavec:
+ intrinsicName = "x86.xsavec";
+ break;
+ case X86::BI__builtin_ia32_xsavec64:
+ intrinsicName = "x86.xsavec64";
+ break;
+ case X86::BI__builtin_ia32_xsaves:
+ intrinsicName = "x86.xsaves";
+ break;
+ case X86::BI__builtin_ia32_xsaves64:
+ intrinsicName = "x86.xsaves64";
+ break;
+ case X86::BI__builtin_ia32_xsetbv:
+ case X86::BI_xsetbv:
+ intrinsicName = "x86.xsetbv";
+ break;
+ }
+
+ // The xsave family of instructions take a 64-bit mask that specifies
+ // which processor state components to save/restore. The hardware expects
+ // this mask split into two 32-bit registers: EDX (high 32 bits) and
+ // EAX (low 32 bits).
+ mlir::Type i32Ty = builder.getSInt32Ty();
+
+ // Mhi = (uint32_t)(ops[1] >> 32) - extract high 32 bits via right shift
+ cir::ConstantOp shift32 = builder.getSInt64(32, loc);
+ mlir::Value mhi = builder.createShift(loc, ops[1], shift32.getResult(),
+ /*isShiftLeft=*/false);
+ mhi = builder.createIntCast(mhi, i32Ty);
+
+ // Mlo = (uint32_t)ops[1] - extract low 32 bits by truncation
+ mlir::Value mlo = builder.createIntCast(ops[1], i32Ty);
+
+ return emitIntrinsicCallOp(builder, loc, intrinsicName, voidTy,
+ mlir::ValueRange{ops[0], mhi, mlo});
+ }
+ case X86::BI__builtin_ia32_xgetbv:
+ case X86::BI_xgetbv:
+ // xgetbv reads the extended control register specified by ops[0] (ECX)
+ // and returns the 64-bit value
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "x86.xgetbv", builder.getUInt64Ty(), ops[0]);
+ case X86::BI__builtin_ia32_storedqudi128_mask:
+ case X86::BI__builtin_ia32_storedqusi128_mask:
+ case X86::BI__builtin_ia32_storedquhi128_mask:
+ case X86::BI__builtin_ia32_storedquqi128_mask:
+ case X86::BI__builtin_ia32_storeupd128_mask:
+ case X86::BI__builtin_ia32_storeups128_mask:
+ case X86::BI__builtin_ia32_storedqudi256_mask:
+ case X86::BI__builtin_ia32_storedqusi256_mask:
+ case X86::BI__builtin_ia32_storedquhi256_mask:
+ case X86::BI__builtin_ia32_storedquqi256_mask:
+ case X86::BI__builtin_ia32_storeupd256_mask:
+ case X86::BI__builtin_ia32_storeups256_mask:
+ case X86::BI__builtin_ia32_storedqudi512_mask:
+ case X86::BI__builtin_ia32_storedqusi512_mask:
+ case X86::BI__builtin_ia32_storedquhi512_mask:
+ case X86::BI__builtin_ia32_storedquqi512_mask:
+ case X86::BI__builtin_ia32_storeupd512_mask:
+ case X86::BI__builtin_ia32_storeups512_mask:
+ case X86::BI__builtin_ia32_storesbf16128_mask:
+ case X86::BI__builtin_ia32_storesh128_mask:
+ case X86::BI__builtin_ia32_storess128_mask:
+ case X86::BI__builtin_ia32_storesd128_mask:
+ case X86::BI__builtin_ia32_cvtmask2b128:
+ case X86::BI__builtin_ia32_cvtmask2b256:
+ case X86::BI__builtin_ia32_cvtmask2b512:
+ case X86::BI__builtin_ia32_cvtmask2w128:
+ case X86::BI__builtin_ia32_cvtmask2w256:
+ case X86::BI__builtin_ia32_cvtmask2w512:
+ case X86::BI__builtin_ia32_cvtmask2d128:
+ case X86::BI__builtin_ia32_cvtmask2d256:
+ case X86::BI__builtin_ia32_cvtmask2d512:
+ case X86::BI__builtin_ia32_cvtmask2q128:
+ case X86::BI__builtin_ia32_cvtmask2q256:
+ case X86::BI__builtin_ia32_cvtmask2q512:
+ case X86::BI__builtin_ia32_cvtb2mask128:
+ case X86::BI__builtin_ia32_cvtb2mask256:
+ case X86::BI__builtin_ia32_cvtb2mask512:
+ case X86::BI__builtin_ia32_cvtw2mask128:
+ case X86::BI__builtin_ia32_cvtw2mask256:
+ case X86::BI__builtin_ia32_cvtw2mask512:
+ case X86::BI__builtin_ia32_cvtd2mask128:
+ case X86::BI__builtin_ia32_cvtd2mask256:
+ case X86::BI__builtin_ia32_cvtd2mask512:
+ case X86::BI__builtin_ia32_cvtq2mask128:
+ case X86::BI__builtin_ia32_cvtq2mask256:
+ case X86::BI__builtin_ia32_cvtq2mask512:
+ case X86::BI__builtin_ia32_cvtdq2ps512_mask:
+ case X86::BI__builtin_ia32_cvtqq2ps512_mask:
+ case X86::BI__builtin_ia32_cvtqq2pd512_mask:
+ case X86::BI__builtin_ia32_vcvtw2ph512_mask:
+ case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
+ case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
+ case X86::BI__builtin_ia32_cvtudq2ps512_mask:
+ case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
+ case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
+ case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
+ case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
+ case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
+ case X86::BI__builtin_ia32_vfmaddsh3_mask:
+ case X86::BI__builtin_ia32_vfmaddss3_mask:
+ case X86::BI__builtin_ia32_vfmaddsd3_mask:
+ case X86::BI__builtin_ia32_vfmaddsh3_maskz:
+ case X86::BI__builtin_ia32_vfmaddss3_maskz:
+ case X86::BI__builtin_ia32_vfmaddsd3_maskz:
+ case X86::BI__builtin_ia32_vfmaddsh3_mask3:
+ case X86::BI__builtin_ia32_vfmaddss3_mask3:
+ case X86::BI__builtin_ia32_vfmaddsd3_mask3:
+ case X86::BI__builtin_ia32_vfmsubsh3_mask3:
+ case X86::BI__builtin_ia32_vfmsubss3_mask3:
+ case X86::BI__builtin_ia32_vfmsubsd3_mask3:
+ case X86::BI__builtin_ia32_vfmaddph512_mask:
+ case X86::BI__builtin_ia32_vfmaddph512_maskz:
+ case X86::BI__builtin_ia32_vfmaddph512_mask3:
+ case X86::BI__builtin_ia32_vfmaddps512_mask:
+ case X86::BI__builtin_ia32_vfmaddps512_maskz:
+ case X86::BI__builtin_ia32_vfmaddps512_mask3:
+ case X86::BI__builtin_ia32_vfmsubps512_mask3:
+ case X86::BI__builtin_ia32_vfmaddpd512_mask:
+ case X86::BI__builtin_ia32_vfmaddpd512_maskz:
+ case X86::BI__builtin_ia32_vfmaddpd512_mask3:
+ case X86::BI__builtin_ia32_vfmsubpd512_mask3:
+ case X86::BI__builtin_ia32_vfmsubph512_mask3:
+ case X86::BI__builtin_ia32_vfmaddsubph512_mask:
+ case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
+ case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
+ case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
+ case X86::BI__builtin_ia32_vfmaddsubps512_mask:
+ case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
+ case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
+ case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
+ case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
+ case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
+ case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
+ case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
+ case X86::BI__builtin_ia32_movdqa32store128_mask:
+ case X86::BI__builtin_ia32_movdqa64store128_mask:
+ case X86::BI__builtin_ia32_storeaps128_mask:
+ case X86::BI__builtin_ia32_storeapd128_mask:
+ case X86::BI__builtin_ia32_movdqa32store256_mask:
+ case X86::BI__builtin_ia32_movdqa64store256_mask:
+ case X86::BI__builtin_ia32_storeaps256_mask:
+ case X86::BI__builtin_ia32_storeapd256_mask:
+ case X86::BI__builtin_ia32_movdqa32store512_mask:
+ case X86::BI__builtin_ia32_movdqa64store512_mask:
+ case X86::BI__builtin_ia32_storeaps512_mask:
+ case X86::BI__builtin_ia32_storeapd512_mask:
+ case X86::BI__builtin_ia32_loadups128_mask:
+ case X86::BI__builtin_ia32_loadups256_mask:
+ case X86::BI__builtin_ia32_loadups512_mask:
+ case X86::BI__builtin_ia32_loadupd128_mask:
+ case X86::BI__builtin_ia32_loadupd256_mask:
+ case X86::BI__builtin_ia32_loadupd512_mask:
+ case X86::BI__builtin_ia32_loaddquqi128_mask:
+ case X86::BI__builtin_ia32_loaddquqi256_mask:
+ case X86::BI__builtin_ia32_loaddquqi512_mask:
+ case X86::BI__builtin_ia32_loaddquhi128_mask:
+ case X86::BI__builtin_ia32_loaddquhi256_mask:
+ case X86::BI__builtin_ia32_loaddquhi512_mask:
+ case X86::BI__builtin_ia32_loaddqusi128_mask:
+ case X86::BI__builtin_ia32_loaddqusi256_mask:
+ case X86::BI__builtin_ia32_loaddqusi512_mask:
+ case X86::BI__builtin_ia32_loaddqudi128_mask:
+ case X86::BI__builtin_ia32_loaddqudi256_mask:
+ case X86::BI__builtin_ia32_loaddqudi512_mask:
+ case X86::BI__builtin_ia32_loadsbf16128_mask:
+ case X86::BI__builtin_ia32_loadsh128_mask:
+ case X86::BI__builtin_ia32_loadss128_mask:
+ case X86::BI__builtin_ia32_loadsd128_mask:
+ case X86::BI__builtin_ia32_loadaps128_mask:
+ case X86::BI__builtin_ia32_loadaps256_mask:
+ case X86::BI__builtin_ia32_loadaps512_mask:
+ case X86::BI__builtin_ia32_loadapd128_mask:
+ case X86::BI__builtin_ia32_loadapd256_mask:
+ case X86::BI__builtin_ia32_loadapd512_mask:
+ case X86::BI__builtin_ia32_movdqa32load128_mask:
+ case X86::BI__builtin_ia32_movdqa32load256_mask:
+ case X86::BI__builtin_ia32_movdqa32load512_mask:
+ case X86::BI__builtin_ia32_movdqa64load128_mask:
+ case X86::BI__builtin_ia32_movdqa64load256_mask:
+ case X86::BI__builtin_ia32_movdqa64load512_mask:
+ case X86::BI__builtin_ia32_expandloaddf128_mask:
+ case X86::BI__builtin_ia32_expandloaddf256_mask:
+ case X86::BI__builtin_ia32_expandloaddf512_mask:
+ case X86::BI__builtin_ia32_expandloadsf128_mask:
+ case X86::BI__builtin_ia32_expandloadsf256_mask:
+ case X86::BI__builtin_ia32_expandloadsf512_mask:
+ case X86::BI__builtin_ia32_expandloaddi128_mask:
+ case X86::BI__builtin_ia32_expandloaddi256_mask:
+ case X86::BI__builtin_ia32_expandloaddi512_mask:
+ case X86::BI__builtin_ia32_expandloadsi128_mask:
+ case X86::BI__builtin_ia32_expandloadsi256_mask:
+ case X86::BI__builtin_ia32_expandloadsi512_mask:
+ case X86::BI__builtin_ia32_expandloadhi128_mask:
+ case X86::BI__builtin_ia32_expandloadhi256_mask:
+ case X86::BI__builtin_ia32_expandloadhi512_mask:
+ case X86::BI__builtin_ia32_expandloadqi128_mask:
+ case X86::BI__builtin_ia32_expandloadqi256_mask:
+ case X86::BI__builtin_ia32_expandloadqi512_mask:
+ case X86::BI__builtin_ia32_compressstoredf128_mask:
+ case X86::BI__builtin_ia32_compressstoredf256_mask:
+ case X86::BI__builtin_ia32_compressstoredf512_mask:
+ case X86::BI__builtin_ia32_compressstoresf128_mask:
+ case X86::BI__builtin_ia32_compressstoresf256_mask:
+ case X86::BI__builtin_ia32_compressstoresf512_mask:
+ case X86::BI__builtin_ia32_compressstoredi128_mask:
+ case X86::BI__builtin_ia32_compressstoredi256_mask:
+ case X86::BI__builtin_ia32_compressstoredi512_mask:
+ case X86::BI__builtin_ia32_compressstoresi128_mask:
+ case X86::BI__builtin_ia32_compressstoresi256_mask:
+ case X86::BI__builtin_ia32_compressstoresi512_mask:
+ case X86::BI__builtin_ia32_compressstorehi128_mask:
+ case X86::BI__builtin_ia32_compressstorehi256_mask:
+ case X86::BI__builtin_ia32_compressstorehi512_mask:
+ case X86::BI__builtin_ia32_compressstoreqi128_mask:
+ case X86::BI__builtin_ia32_compressstoreqi256_mask:
+ case X86::BI__builtin_ia32_compressstoreqi512_mask:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_expanddf128_mask:
+ case X86::BI__builtin_ia32_expanddf256_mask:
+ case X86::BI__builtin_ia32_expanddf512_mask:
+ case X86::BI__builtin_ia32_expandsf128_mask:
+ case X86::BI__builtin_ia32_expandsf256_mask:
+ case X86::BI__builtin_ia32_expandsf512_mask:
+ case X86::BI__builtin_ia32_expanddi128_mask:
+ case X86::BI__builtin_ia32_expanddi256_mask:
+ case X86::BI__builtin_ia32_expanddi512_mask:
+ case X86::BI__builtin_ia32_expandsi128_mask:
+ case X86::BI__builtin_ia32_expandsi256_mask:
+ case X86::BI__builtin_ia32_expandsi512_mask:
+ case X86::BI__builtin_ia32_expandhi128_mask:
+ case X86::BI__builtin_ia32_expandhi256_mask:
+ case X86::BI__builtin_ia32_expandhi512_mask:
+ case X86::BI__builtin_ia32_expandqi128_mask:
+ case X86::BI__builtin_ia32_expandqi256_mask:
+ case X86::BI__builtin_ia32_expandqi512_mask: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ return emitX86CompressExpand(builder, loc, ops[0], ops[1], ops[2],
+ "x86.avx512.mask.expand");
+ }
+ case X86::BI__builtin_ia32_compressdf128_mask:
+ case X86::BI__builtin_ia32_compressdf256_mask:
+ case X86::BI__builtin_ia32_compressdf512_mask:
+ case X86::BI__builtin_ia32_compresssf128_mask:
+ case X86::BI__builtin_ia32_compresssf256_mask:
+ case X86::BI__builtin_ia32_compresssf512_mask:
+ case X86::BI__builtin_ia32_compressdi128_mask:
+ case X86::BI__builtin_ia32_compressdi256_mask:
+ case X86::BI__builtin_ia32_compressdi512_mask:
+ case X86::BI__builtin_ia32_compresssi128_mask:
+ case X86::BI__builtin_ia32_compresssi256_mask:
+ case X86::BI__builtin_ia32_compresssi512_mask:
+ case X86::BI__builtin_ia32_compresshi128_mask:
+ case X86::BI__builtin_ia32_compresshi256_mask:
+ case X86::BI__builtin_ia32_compresshi512_mask:
+ case X86::BI__builtin_ia32_compressqi128_mask:
+ case X86::BI__builtin_ia32_compressqi256_mask:
+ case X86::BI__builtin_ia32_compressqi512_mask: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ return emitX86CompressExpand(builder, loc, ops[0], ops[1], ops[2],
+ "x86.avx512.mask.compress");
}
-
- // The xsave family of instructions take a 64-bit mask that specifies
- // which processor state components to save/restore. The hardware expects
- // this mask split into two 32-bit registers: EDX (high 32 bits) and
- // EAX (low 32 bits).
- mlir::Type i32Ty = builder.getSInt32Ty();
-
- // Mhi = (uint32_t)(ops[1] >> 32) - extract high 32 bits via right shift
- cir::ConstantOp shift32 = builder.getSInt64(32, loc);
- mlir::Value mhi = builder.createShift(loc, ops[1], shift32.getResult(),
- /*isShiftLeft=*/false);
- mhi = builder.createIntCast(mhi, i32Ty);
-
- // Mlo = (uint32_t)ops[1] - extract low 32 bits by truncation
- mlir::Value mlo = builder.createIntCast(ops[1], i32Ty);
-
- return emitIntrinsicCallOp(builder, loc, intrinsicName, voidTy,
- mlir::ValueRange{ops[0], mhi, mlo});
- }
- case X86::BI__builtin_ia32_xgetbv:
- case X86::BI_xgetbv:
- // xgetbv reads the extended control register specified by ops[0] (ECX)
- // and returns the 64-bit value
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "x86.xgetbv", builder.getUInt64Ty(), ops[0]);
- case X86::BI__builtin_ia32_storedqudi128_mask:
- case X86::BI__builtin_ia32_storedqusi128_mask:
- case X86::BI__builtin_ia32_storedquhi128_mask:
- case X86::BI__builtin_ia32_storedquqi128_mask:
- case X86::BI__builtin_ia32_storeupd128_mask:
- case X86::BI__builtin_ia32_storeups128_mask:
- case X86::BI__builtin_ia32_storedqudi256_mask:
- case X86::BI__builtin_ia32_storedqusi256_mask:
- case X86::BI__builtin_ia32_storedquhi256_mask:
- case X86::BI__builtin_ia32_storedquqi256_mask:
- case X86::BI__builtin_ia32_storeupd256_mask:
- case X86::BI__builtin_ia32_storeups256_mask:
- case X86::BI__builtin_ia32_storedqudi512_mask:
- case X86::BI__builtin_ia32_storedqusi512_mask:
- case X86::BI__builtin_ia32_storedquhi512_mask:
- case X86::BI__builtin_ia32_storedquqi512_mask:
- case X86::BI__builtin_ia32_storeupd512_mask:
- case X86::BI__builtin_ia32_storeups512_mask:
- case X86::BI__builtin_ia32_storesbf16128_mask:
- case X86::BI__builtin_ia32_storesh128_mask:
- case X86::BI__builtin_ia32_storess128_mask:
- case X86::BI__builtin_ia32_storesd128_mask:
- case X86::BI__builtin_ia32_cvtmask2b128:
- case X86::BI__builtin_ia32_cvtmask2b256:
- case X86::BI__builtin_ia32_cvtmask2b512:
- case X86::BI__builtin_ia32_cvtmask2w128:
- case X86::BI__builtin_ia32_cvtmask2w256:
- case X86::BI__builtin_ia32_cvtmask2w512:
- case X86::BI__builtin_ia32_cvtmask2d128:
- case X86::BI__builtin_ia32_cvtmask2d256:
- case X86::BI__builtin_ia32_cvtmask2d512:
- case X86::BI__builtin_ia32_cvtmask2q128:
- case X86::BI__builtin_ia32_cvtmask2q256:
- case X86::BI__builtin_ia32_cvtmask2q512:
- case X86::BI__builtin_ia32_cvtb2mask128:
- case X86::BI__builtin_ia32_cvtb2mask256:
- case X86::BI__builtin_ia32_cvtb2mask512:
- case X86::BI__builtin_ia32_cvtw2mask128:
- case X86::BI__builtin_ia32_cvtw2mask256:
- case X86::BI__builtin_ia32_cvtw2mask512:
- case X86::BI__builtin_ia32_cvtd2mask128:
- case X86::BI__builtin_ia32_cvtd2mask256:
- case X86::BI__builtin_ia32_cvtd2mask512:
- case X86::BI__builtin_ia32_cvtq2mask128:
- case X86::BI__builtin_ia32_cvtq2mask256:
- case X86::BI__builtin_ia32_cvtq2mask512:
- case X86::BI__builtin_ia32_cvtdq2ps512_mask:
- case X86::BI__builtin_ia32_cvtqq2ps512_mask:
- case X86::BI__builtin_ia32_cvtqq2pd512_mask:
- case X86::BI__builtin_ia32_vcvtw2ph512_mask:
- case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
- case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
- case X86::BI__builtin_ia32_cvtudq2ps512_mask:
- case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
- case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
- case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
- case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
- case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
- case X86::BI__builtin_ia32_vfmaddsh3_mask:
- case X86::BI__builtin_ia32_vfmaddss3_mask:
- case X86::BI__builtin_ia32_vfmaddsd3_mask:
- case X86::BI__builtin_ia32_vfmaddsh3_maskz:
- case X86::BI__builtin_ia32_vfmaddss3_maskz:
- case X86::BI__builtin_ia32_vfmaddsd3_maskz:
- case X86::BI__builtin_ia32_vfmaddsh3_mask3:
- case X86::BI__builtin_ia32_vfmaddss3_mask3:
- case X86::BI__builtin_ia32_vfmaddsd3_mask3:
- case X86::BI__builtin_ia32_vfmsubsh3_mask3:
- case X86::BI__builtin_ia32_vfmsubss3_mask3:
- case X86::BI__builtin_ia32_vfmsubsd3_mask3:
- case X86::BI__builtin_ia32_vfmaddph512_mask:
- case X86::BI__builtin_ia32_vfmaddph512_maskz:
- case X86::BI__builtin_ia32_vfmaddph512_mask3:
- case X86::BI__builtin_ia32_vfmaddps512_mask:
- case X86::BI__builtin_ia32_vfmaddps512_maskz:
- case X86::BI__builtin_ia32_vfmaddps512_mask3:
- case X86::BI__builtin_ia32_vfmsubps512_mask3:
- case X86::BI__builtin_ia32_vfmaddpd512_mask:
- case X86::BI__builtin_ia32_vfmaddpd512_maskz:
- case X86::BI__builtin_ia32_vfmaddpd512_mask3:
- case X86::BI__builtin_ia32_vfmsubpd512_mask3:
- case X86::BI__builtin_ia32_vfmsubph512_mask3:
- case X86::BI__builtin_ia32_vfmaddsubph512_mask:
- case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
- case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
- case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
- case X86::BI__builtin_ia32_vfmaddsubps512_mask:
- case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
- case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
- case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
- case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
- case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
- case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
- case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
- case X86::BI__builtin_ia32_movdqa32store128_mask:
- case X86::BI__builtin_ia32_movdqa64store128_mask:
- case X86::BI__builtin_ia32_storeaps128_mask:
- case X86::BI__builtin_ia32_storeapd128_mask:
- case X86::BI__builtin_ia32_movdqa32store256_mask:
- case X86::BI__builtin_ia32_movdqa64store256_mask:
- case X86::BI__builtin_ia32_storeaps256_mask:
- case X86::BI__builtin_ia32_storeapd256_mask:
- case X86::BI__builtin_ia32_movdqa32store512_mask:
- case X86::BI__builtin_ia32_movdqa64store512_mask:
- case X86::BI__builtin_ia32_storeaps512_mask:
- case X86::BI__builtin_ia32_storeapd512_mask:
- case X86::BI__builtin_ia32_loadups128_mask:
- case X86::BI__builtin_ia32_loadups256_mask:
- case X86::BI__builtin_ia32_loadups512_mask:
- case X86::BI__builtin_ia32_loadupd128_mask:
- case X86::BI__builtin_ia32_loadupd256_mask:
- case X86::BI__builtin_ia32_loadupd512_mask:
- case X86::BI__builtin_ia32_loaddquqi128_mask:
- case X86::BI__builtin_ia32_loaddquqi256_mask:
- case X86::BI__builtin_ia32_loaddquqi512_mask:
- case X86::BI__builtin_ia32_loaddquhi128_mask:
- case X86::BI__builtin_ia32_loaddquhi256_mask:
- case X86::BI__builtin_ia32_loaddquhi512_mask:
- case X86::BI__builtin_ia32_loaddqusi128_mask:
- case X86::BI__builtin_ia32_loaddqusi256_mask:
- case X86::BI__builtin_ia32_loaddqusi512_mask:
- case X86::BI__builtin_ia32_loaddqudi128_mask:
- case X86::BI__builtin_ia32_loaddqudi256_mask:
- case X86::BI__builtin_ia32_loaddqudi512_mask:
- case X86::BI__builtin_ia32_loadsbf16128_mask:
- case X86::BI__builtin_ia32_loadsh128_mask:
- case X86::BI__builtin_ia32_loadss128_mask:
- case X86::BI__builtin_ia32_loadsd128_mask:
- case X86::BI__builtin_ia32_loadaps128_mask:
- case X86::BI__builtin_ia32_loadaps256_mask:
- case X86::BI__builtin_ia32_loadaps512_mask:
- case X86::BI__builtin_ia32_loadapd128_mask:
- case X86::BI__builtin_ia32_loadapd256_mask:
- case X86::BI__builtin_ia32_loadapd512_mask:
- case X86::BI__builtin_ia32_movdqa32load128_mask:
- case X86::BI__builtin_ia32_movdqa32load256_mask:
- case X86::BI__builtin_ia32_movdqa32load512_mask:
- case X86::BI__builtin_ia32_movdqa64load128_mask:
- case X86::BI__builtin_ia32_movdqa64load256_mask:
- case X86::BI__builtin_ia32_movdqa64load512_mask:
- case X86::BI__builtin_ia32_expandloaddf128_mask:
- case X86::BI__builtin_ia32_expandloaddf256_mask:
- case X86::BI__builtin_ia32_expandloaddf512_mask:
- case X86::BI__builtin_ia32_expandloadsf128_mask:
- case X86::BI__builtin_ia32_expandloadsf256_mask:
- case X86::BI__builtin_ia32_expandloadsf512_mask:
- case X86::BI__builtin_ia32_expandloaddi128_mask:
- case X86::BI__builtin_ia32_expandloaddi256_mask:
- case X86::BI__builtin_ia32_expandloaddi512_mask:
- case X86::BI__builtin_ia32_expandloadsi128_mask:
- case X86::BI__builtin_ia32_expandloadsi256_mask:
- case X86::BI__builtin_ia32_expandloadsi512_mask:
- case X86::BI__builtin_ia32_expandloadhi128_mask:
- case X86::BI__builtin_ia32_expandloadhi256_mask:
- case X86::BI__builtin_ia32_expandloadhi512_mask:
- case X86::BI__builtin_ia32_expandloadqi128_mask:
- case X86::BI__builtin_ia32_expandloadqi256_mask:
- case X86::BI__builtin_ia32_expandloadqi512_mask:
- case X86::BI__builtin_ia32_compressstoredf128_mask:
- case X86::BI__builtin_ia32_compressstoredf256_mask:
- case X86::BI__builtin_ia32_compressstoredf512_mask:
- case X86::BI__builtin_ia32_compressstoresf128_mask:
- case X86::BI__builtin_ia32_compressstoresf256_mask:
- case X86::BI__builtin_ia32_compressstoresf512_mask:
- case X86::BI__builtin_ia32_compressstoredi128_mask:
- case X86::BI__builtin_ia32_compressstoredi256_mask:
- case X86::BI__builtin_ia32_compressstoredi512_mask:
- case X86::BI__builtin_ia32_compressstoresi128_mask:
- case X86::BI__builtin_ia32_compressstoresi256_mask:
- case X86::BI__builtin_ia32_compressstoresi512_mask:
- case X86::BI__builtin_ia32_compressstorehi128_mask:
- case X86::BI__builtin_ia32_compressstorehi256_mask:
- case X86::BI__builtin_ia32_compressstorehi512_mask:
- case X86::BI__builtin_ia32_compressstoreqi128_mask:
- case X86::BI__builtin_ia32_compressstoreqi256_mask:
- case X86::BI__builtin_ia32_compressstoreqi512_mask:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_expanddf128_mask:
- case X86::BI__builtin_ia32_expanddf256_mask:
- case X86::BI__builtin_ia32_expanddf512_mask:
- case X86::BI__builtin_ia32_expandsf128_mask:
- case X86::BI__builtin_ia32_expandsf256_mask:
- case X86::BI__builtin_ia32_expandsf512_mask:
- case X86::BI__builtin_ia32_expanddi128_mask:
- case X86::BI__builtin_ia32_expanddi256_mask:
- case X86::BI__builtin_ia32_expanddi512_mask:
- case X86::BI__builtin_ia32_expandsi128_mask:
- case X86::BI__builtin_ia32_expandsi256_mask:
- case X86::BI__builtin_ia32_expandsi512_mask:
- case X86::BI__builtin_ia32_expandhi128_mask:
- case X86::BI__builtin_ia32_expandhi256_mask:
- case X86::BI__builtin_ia32_expandhi512_mask:
- case X86::BI__builtin_ia32_expandqi128_mask:
- case X86::BI__builtin_ia32_expandqi256_mask:
- case X86::BI__builtin_ia32_expandqi512_mask: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- return emitX86CompressExpand(builder, loc, ops[0], ops[1], ops[2],
- "x86.avx512.mask.expand");
- }
- case X86::BI__builtin_ia32_compressdf128_mask:
- case X86::BI__builtin_ia32_compressdf256_mask:
- case X86::BI__builtin_ia32_compressdf512_mask:
- case X86::BI__builtin_ia32_compresssf128_mask:
- case X86::BI__builtin_ia32_compresssf256_mask:
- case X86::BI__builtin_ia32_compresssf512_mask:
- case X86::BI__builtin_ia32_compressdi128_mask:
- case X86::BI__builtin_ia32_compressdi256_mask:
- case X86::BI__builtin_ia32_compressdi512_mask:
- case X86::BI__builtin_ia32_compresssi128_mask:
- case X86::BI__builtin_ia32_compresssi256_mask:
- case X86::BI__builtin_ia32_compresssi512_mask:
- case X86::BI__builtin_ia32_compresshi128_mask:
- case X86::BI__builtin_ia32_compresshi256_mask:
- case X86::BI__builtin_ia32_compresshi512_mask:
- case X86::BI__builtin_ia32_compressqi128_mask:
- case X86::BI__builtin_ia32_compressqi256_mask:
- case X86::BI__builtin_ia32_compressqi512_mask: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- return emitX86CompressExpand(builder, loc, ops[0], ops[1], ops[2],
- "x86.avx512.mask.compress");
- }
- case X86::BI__builtin_ia32_gather3div2df:
- case X86::BI__builtin_ia32_gather3div2di:
- case X86::BI__builtin_ia32_gather3div4df:
- case X86::BI__builtin_ia32_gather3div4di:
- case X86::BI__builtin_ia32_gather3div4sf:
- case X86::BI__builtin_ia32_gather3div4si:
- case X86::BI__builtin_ia32_gather3div8sf:
- case X86::BI__builtin_ia32_gather3div8si:
- case X86::BI__builtin_ia32_gather3siv2df:
- case X86::BI__builtin_ia32_gather3siv2di:
- case X86::BI__builtin_ia32_gather3siv4df:
- case X86::BI__builtin_ia32_gather3siv4di:
- case X86::BI__builtin_ia32_gather3siv4sf:
- case X86::BI__builtin_ia32_gather3siv4si:
- case X86::BI__builtin_ia32_gather3siv8sf:
- case X86::BI__builtin_ia32_gather3siv8si:
- case X86::BI__builtin_ia32_gathersiv8df:
- case X86::BI__builtin_ia32_gathersiv16sf:
- case X86::BI__builtin_ia32_gatherdiv8df:
- case X86::BI__builtin_ia32_gatherdiv16sf:
- case X86::BI__builtin_ia32_gathersiv8di:
- case X86::BI__builtin_ia32_gathersiv16si:
- case X86::BI__builtin_ia32_gatherdiv8di:
- case X86::BI__builtin_ia32_gatherdiv16si: {
- StringRef intrinsicName;
- switch (builtinID) {
- default:
- llvm_unreachable("Unexpected builtin");
case X86::BI__builtin_ia32_gather3div2df:
- intrinsicName = "x86.avx512.mask.gather3div2.df";
- break;
case X86::BI__builtin_ia32_gather3div2di:
- intrinsicName = "x86.avx512.mask.gather3div2.di";
- break;
case X86::BI__builtin_ia32_gather3div4df:
- intrinsicName = "x86.avx512.mask.gather3div4.df";
- break;
case X86::BI__builtin_ia32_gather3div4di:
- intrinsicName = "x86.avx512.mask.gather3div4.di";
- break;
case X86::BI__builtin_ia32_gather3div4sf:
- intrinsicName = "x86.avx512.mask.gather3div4.sf";
- break;
case X86::BI__builtin_ia32_gather3div4si:
- intrinsicName = "x86.avx512.mask.gather3div4.si";
- break;
case X86::BI__builtin_ia32_gather3div8sf:
- intrinsicName = "x86.avx512.mask.gather3div8.sf";
- break;
case X86::BI__builtin_ia32_gather3div8si:
- intrinsicName = "x86.avx512.mask.gather3div8.si";
- break;
case X86::BI__builtin_ia32_gather3siv2df:
- intrinsicName = "x86.avx512.mask.gather3siv2.df";
- break;
case X86::BI__builtin_ia32_gather3siv2di:
- intrinsicName = "x86.avx512.mask.gather3siv2.di";
- break;
case X86::BI__builtin_ia32_gather3siv4df:
- intrinsicName = "x86.avx512.mask.gather3siv4.df";
- break;
case X86::BI__builtin_ia32_gather3siv4di:
- intrinsicName = "x86.avx512.mask.gather3siv4.di";
- break;
case X86::BI__builtin_ia32_gather3siv4sf:
- intrinsicName = "x86.avx512.mask.gather3siv4.sf";
- break;
case X86::BI__builtin_ia32_gather3siv4si:
- intrinsicName = "x86.avx512.mask.gather3siv4.si";
- break;
case X86::BI__builtin_ia32_gather3siv8sf:
- intrinsicName = "x86.avx512.mask.gather3siv8.sf";
- break;
case X86::BI__builtin_ia32_gather3siv8si:
- intrinsicName = "x86.avx512.mask.gather3siv8.si";
- break;
case X86::BI__builtin_ia32_gathersiv8df:
- intrinsicName = "x86.avx512.mask.gather.dpd.512";
- break;
case X86::BI__builtin_ia32_gathersiv16sf:
- intrinsicName = "x86.avx512.mask.gather.dps.512";
- break;
case X86::BI__builtin_ia32_gatherdiv8df:
- intrinsicName = "x86.avx512.mask.gather.qpd.512";
- break;
case X86::BI__builtin_ia32_gatherdiv16sf:
- intrinsicName = "x86.avx512.mask.gather.qps.512";
- break;
case X86::BI__builtin_ia32_gathersiv8di:
- intrinsicName = "x86.avx512.mask.gather.dpq.512";
- break;
case X86::BI__builtin_ia32_gathersiv16si:
- intrinsicName = "x86.avx512.mask.gather.dpi.512";
- break;
case X86::BI__builtin_ia32_gatherdiv8di:
- intrinsicName = "x86.avx512.mask.gather.qpq.512";
- break;
- case X86::BI__builtin_ia32_gatherdiv16si:
- intrinsicName = "x86.avx512.mask.gather.qpi.512";
- break;
+ case X86::BI__builtin_ia32_gatherdiv16si: {
+ StringRef intrinsicName;
+ switch (builtinID) {
+ default:
+ llvm_unreachable("Unexpected builtin");
+ case X86::BI__builtin_ia32_gather3div2df:
+ intrinsicName = "x86.avx512.mask.gather3div2.df";
+ break;
+ case X86::BI__builtin_ia32_gather3div2di:
+ intrinsicName = "x86.avx512.mask.gather3div2.di";
+ break;
+ case X86::BI__builtin_ia32_gather3div4df:
+ intrinsicName = "x86.avx512.mask.gather3div4.df";
+ break;
+ case X86::BI__builtin_ia32_gather3div4di:
+ intrinsicName = "x86.avx512.mask.gather3div4.di";
+ break;
+ case X86::BI__builtin_ia32_gather3div4sf:
+ intrinsicName = "x86.avx512.mask.gather3div4.sf";
+ break;
+ case X86::BI__builtin_ia32_gather3div4si:
+ intrinsicName = "x86.avx512.mask.gather3div4.si";
+ break;
+ case X86::BI__builtin_ia32_gather3div8sf:
+ intrinsicName = "x86.avx512.mask.gather3div8.sf";
+ break;
+ case X86::BI__builtin_ia32_gather3div8si:
+ intrinsicName = "x86.avx512.mask.gather3div8.si";
+ break;
+ case X86::BI__builtin_ia32_gather3siv2df:
+ intrinsicName = "x86.avx512.mask.gather3siv2.df";
+ break;
+ case X86::BI__builtin_ia32_gather3siv2di:
+ intrinsicName = "x86.avx512.mask.gather3siv2.di";
+ break;
+ case X86::BI__builtin_ia32_gather3siv4df:
+ intrinsicName = "x86.avx512.mask.gather3siv4.df";
+ break;
+ case X86::BI__builtin_ia32_gather3siv4di:
+ intrinsicName = "x86.avx512.mask.gather3siv4.di";
+ break;
+ case X86::BI__builtin_ia32_gather3siv4sf:
+ intrinsicName = "x86.avx512.mask.gather3siv4.sf";
+ break;
+ case X86::BI__builtin_ia32_gather3siv4si:
+ intrinsicName = "x86.avx512.mask.gather3siv4.si";
+ break;
+ case X86::BI__builtin_ia32_gather3siv8sf:
+ intrinsicName = "x86.avx512.mask.gather3siv8.sf";
+ break;
+ case X86::BI__builtin_ia32_gather3siv8si:
+ intrinsicName = "x86.avx512.mask.gather3siv8.si";
+ break;
+ case X86::BI__builtin_ia32_gathersiv8df:
+ intrinsicName = "x86.avx512.mask.gather.dpd.512";
+ break;
+ case X86::BI__builtin_ia32_gathersiv16sf:
+ intrinsicName = "x86.avx512.mask.gather.dps.512";
+ break;
+ case X86::BI__builtin_ia32_gatherdiv8df:
+ intrinsicName = "x86.avx512.mask.gather.qpd.512";
+ break;
+ case X86::BI__builtin_ia32_gatherdiv16sf:
+ intrinsicName = "x86.avx512.mask.gather.qps.512";
+ break;
+ case X86::BI__builtin_ia32_gathersiv8di:
+ intrinsicName = "x86.avx512.mask.gather.dpq.512";
+ break;
+ case X86::BI__builtin_ia32_gathersiv16si:
+ intrinsicName = "x86.avx512.mask.gather.dpi.512";
+ break;
+ case X86::BI__builtin_ia32_gatherdiv8di:
+ intrinsicName = "x86.avx512.mask.gather.qpq.512";
+ break;
+ case X86::BI__builtin_ia32_gatherdiv16si:
+ intrinsicName = "x86.avx512.mask.gather.qpi.512";
+ break;
+ }
+
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ unsigned minElts =
+ std::min(cast<cir::VectorType>(ops[0].getType()).getSize(),
+ cast<cir::VectorType>(ops[2].getType()).getSize());
+ ops[3] = getMaskVecValue(builder, loc, ops[3], minElts);
+ return emitIntrinsicCallOp(builder, loc, intrinsicName,
+ convertType(expr->getType()), ops);
}
-
- mlir::Location loc = getLoc(expr->getExprLoc());
- unsigned minElts =
- std::min(cast<cir::VectorType>(ops[0].getType()).getSize(),
- cast<cir::VectorType>(ops[2].getType()).getSize());
- ops[3] = getMaskVecValue(builder, loc, ops[3], minElts);
- return emitIntrinsicCallOp(builder, loc, intrinsicName,
- convertType(expr->getType()), ops);
- }
- case X86::BI__builtin_ia32_scattersiv8df:
- case X86::BI__builtin_ia32_scattersiv16sf:
- case X86::BI__builtin_ia32_scatterdiv8df:
- case X86::BI__builtin_ia32_scatterdiv16sf:
- case X86::BI__builtin_ia32_scattersiv8di:
- case X86::BI__builtin_ia32_scattersiv16si:
- case X86::BI__builtin_ia32_scatterdiv8di:
- case X86::BI__builtin_ia32_scatterdiv16si:
- case X86::BI__builtin_ia32_scatterdiv2df:
- case X86::BI__builtin_ia32_scatterdiv2di:
- case X86::BI__builtin_ia32_scatterdiv4df:
- case X86::BI__builtin_ia32_scatterdiv4di:
- case X86::BI__builtin_ia32_scatterdiv4sf:
- case X86::BI__builtin_ia32_scatterdiv4si:
- case X86::BI__builtin_ia32_scatterdiv8sf:
- case X86::BI__builtin_ia32_scatterdiv8si:
- case X86::BI__builtin_ia32_scattersiv2df:
- case X86::BI__builtin_ia32_scattersiv2di:
- case X86::BI__builtin_ia32_scattersiv4df:
- case X86::BI__builtin_ia32_scattersiv4di:
- case X86::BI__builtin_ia32_scattersiv4sf:
- case X86::BI__builtin_ia32_scattersiv4si:
- case X86::BI__builtin_ia32_scattersiv8sf:
- case X86::BI__builtin_ia32_scattersiv8si: {
- llvm::StringRef intrinsicName;
- switch (builtinID) {
- default:
- llvm_unreachable("Unexpected builtin");
case X86::BI__builtin_ia32_scattersiv8df:
- intrinsicName = "x86.avx512.mask.scatter.dpd.512";
- break;
case X86::BI__builtin_ia32_scattersiv16sf:
- intrinsicName = "x86.avx512.mask.scatter.dps.512";
- break;
case X86::BI__builtin_ia32_scatterdiv8df:
- intrinsicName = "x86.avx512.mask.scatter.qpd.512";
- break;
case X86::BI__builtin_ia32_scatterdiv16sf:
- intrinsicName = "x86.avx512.mask.scatter.qps.512";
- break;
case X86::BI__builtin_ia32_scattersiv8di:
- intrinsicName = "x86.avx512.mask.scatter.dpq.512";
- break;
case X86::BI__builtin_ia32_scattersiv16si:
- intrinsicName = "x86.avx512.mask.scatter.dpi.512";
- break;
case X86::BI__builtin_ia32_scatterdiv8di:
- intrinsicName = "x86.avx512.mask.scatter.qpq.512";
- break;
case X86::BI__builtin_ia32_scatterdiv16si:
- intrinsicName = "x86.avx512.mask.scatter.qpi.512";
- break;
case X86::BI__builtin_ia32_scatterdiv2df:
- intrinsicName = "x86.avx512.mask.scatterdiv2.df";
- break;
case X86::BI__builtin_ia32_scatterdiv2di:
- intrinsicName = "x86.avx512.mask.scatterdiv2.di";
- break;
case X86::BI__builtin_ia32_scatterdiv4df:
- intrinsicName = "x86.avx512.mask.scatterdiv4.df";
- break;
case X86::BI__builtin_ia32_scatterdiv4di:
- intrinsicName = "x86.avx512.mask.scatterdiv4.di";
- break;
case X86::BI__builtin_ia32_scatterdiv4sf:
- intrinsicName = "x86.avx512.mask.scatterdiv4.sf";
- break;
case X86::BI__builtin_ia32_scatterdiv4si:
- intrinsicName = "x86.avx512.mask.scatterdiv4.si";
- break;
case X86::BI__builtin_ia32_scatterdiv8sf:
- intrinsicName = "x86.avx512.mask.scatterdiv8.sf";
- break;
case X86::BI__builtin_ia32_scatterdiv8si:
- intrinsicName = "x86.avx512.mask.scatterdiv8.si";
- break;
case X86::BI__builtin_ia32_scattersiv2df:
- intrinsicName = "x86.avx512.mask.scattersiv2.df";
- break;
case X86::BI__builtin_ia32_scattersiv2di:
- intrinsicName = "x86.avx512.mask.scattersiv2.di";
- break;
case X86::BI__builtin_ia32_scattersiv4df:
- intrinsicName = "x86.avx512.mask.scattersiv4.df";
- break;
case X86::BI__builtin_ia32_scattersiv4di:
- intrinsicName = "x86.avx512.mask.scattersiv4.di";
- break;
case X86::BI__builtin_ia32_scattersiv4sf:
- intrinsicName = "x86.avx512.mask.scattersiv4.sf";
- break;
case X86::BI__builtin_ia32_scattersiv4si:
- intrinsicName = "x86.avx512.mask.scattersiv4.si";
- break;
case X86::BI__builtin_ia32_scattersiv8sf:
- intrinsicName = "x86.avx512.mask.scattersiv8.sf";
- break;
- case X86::BI__builtin_ia32_scattersiv8si:
- intrinsicName = "x86.avx512.mask.scattersiv8.si";
- break;
+ case X86::BI__builtin_ia32_scattersiv8si: {
+ llvm::StringRef intrinsicName;
+ switch (builtinID) {
+ default:
+ llvm_unreachable("Unexpected builtin");
+ case X86::BI__builtin_ia32_scattersiv8df:
+ intrinsicName = "x86.avx512.mask.scatter.dpd.512";
+ break;
+ case X86::BI__builtin_ia32_scattersiv16sf:
+ intrinsicName = "x86.avx512.mask.scatter.dps.512";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv8df:
+ intrinsicName = "x86.avx512.mask.scatter.qpd.512";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv16sf:
+ intrinsicName = "x86.avx512.mask.scatter.qps.512";
+ break;
+ case X86::BI__builtin_ia32_scattersiv8di:
+ intrinsicName = "x86.avx512.mask.scatter.dpq.512";
+ break;
+ case X86::BI__builtin_ia32_scattersiv16si:
+ intrinsicName = "x86.avx512.mask.scatter.dpi.512";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv8di:
+ intrinsicName = "x86.avx512.mask.scatter.qpq.512";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv16si:
+ intrinsicName = "x86.avx512.mask.scatter.qpi.512";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv2df:
+ intrinsicName = "x86.avx512.mask.scatterdiv2.df";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv2di:
+ intrinsicName = "x86.avx512.mask.scatterdiv2.di";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv4df:
+ intrinsicName = "x86.avx512.mask.scatterdiv4.df";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv4di:
+ intrinsicName = "x86.avx512.mask.scatterdiv4.di";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv4sf:
+ intrinsicName = "x86.avx512.mask.scatterdiv4.sf";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv4si:
+ intrinsicName = "x86.avx512.mask.scatterdiv4.si";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv8sf:
+ intrinsicName = "x86.avx512.mask.scatterdiv8.sf";
+ break;
+ case X86::BI__builtin_ia32_scatterdiv8si:
+ intrinsicName = "x86.avx512.mask.scatterdiv8.si";
+ break;
+ case X86::BI__builtin_ia32_scattersiv2df:
+ intrinsicName = "x86.avx512.mask.scattersiv2.df";
+ break;
+ case X86::BI__builtin_ia32_scattersiv2di:
+ intrinsicName = "x86.avx512.mask.scattersiv2.di";
+ break;
+ case X86::BI__builtin_ia32_scattersiv4df:
+ intrinsicName = "x86.avx512.mask.scattersiv4.df";
+ break;
+ case X86::BI__builtin_ia32_scattersiv4di:
+ intrinsicName = "x86.avx512.mask.scattersiv4.di";
+ break;
+ case X86::BI__builtin_ia32_scattersiv4sf:
+ intrinsicName = "x86.avx512.mask.scattersiv4.sf";
+ break;
+ case X86::BI__builtin_ia32_scattersiv4si:
+ intrinsicName = "x86.avx512.mask.scattersiv4.si";
+ break;
+ case X86::BI__builtin_ia32_scattersiv8sf:
+ intrinsicName = "x86.avx512.mask.scattersiv8.sf";
+ break;
+ case X86::BI__builtin_ia32_scattersiv8si:
+ intrinsicName = "x86.avx512.mask.scattersiv8.si";
+ break;
+ }
+
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ unsigned minElts =
+ std::min(cast<cir::VectorType>(ops[2].getType()).getSize(),
+ cast<cir::VectorType>(ops[3].getType()).getSize());
+ ops[1] = getMaskVecValue(builder, loc, ops[1], minElts);
+
+ return emitIntrinsicCallOp(builder, loc, intrinsicName,
+ convertType(expr->getType()), ops);
+ }
+ case X86::BI__builtin_ia32_vextractf128_pd256:
+ case X86::BI__builtin_ia32_vextractf128_ps256:
+ case X86::BI__builtin_ia32_vextractf128_si256:
+ case X86::BI__builtin_ia32_extract128i256:
+ case X86::BI__builtin_ia32_extractf64x4_mask:
+ case X86::BI__builtin_ia32_extractf32x4_mask:
+ case X86::BI__builtin_ia32_extracti64x4_mask:
+ case X86::BI__builtin_ia32_extracti32x4_mask:
+ case X86::BI__builtin_ia32_extractf32x8_mask:
+ case X86::BI__builtin_ia32_extracti32x8_mask:
+ case X86::BI__builtin_ia32_extractf32x4_256_mask:
+ case X86::BI__builtin_ia32_extracti32x4_256_mask:
+ case X86::BI__builtin_ia32_extractf64x2_256_mask:
+ case X86::BI__builtin_ia32_extracti64x2_256_mask:
+ case X86::BI__builtin_ia32_extractf64x2_512_mask:
+ case X86::BI__builtin_ia32_extracti64x2_512_mask: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ cir::VectorType dstTy =
+ cast<cir::VectorType>(convertType(expr->getType()));
+ unsigned numElts = dstTy.getSize();
+ unsigned srcNumElts = cast<cir::VectorType>(ops[0].getType()).getSize();
+ unsigned subVectors = srcNumElts / numElts;
+ assert(llvm::isPowerOf2_32(subVectors) &&
+ "Expected power of 2 subvectors");
+ unsigned index =
+ ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
+
+ index &= subVectors - 1; // Remove any extra bits.
+ index *= numElts;
+
+ int64_t indices[16];
+ std::iota(indices, indices + numElts, index);
+
+ mlir::Value poison =
+ builder.getConstant(loc, cir::PoisonAttr::get(ops[0].getType()));
+ mlir::Value res = builder.createVecShuffle(loc, ops[0], poison,
+ ArrayRef(indices, numElts));
+ if (ops.size() == 4)
+ res = emitX86Select(builder, loc, ops[3], res, ops[2]);
+
+ return res;
+ }
+ case X86::BI__builtin_ia32_vinsertf128_pd256:
+ case X86::BI__builtin_ia32_vinsertf128_ps256:
+ case X86::BI__builtin_ia32_vinsertf128_si256:
+ case X86::BI__builtin_ia32_insert128i256:
+ case X86::BI__builtin_ia32_insertf64x4:
+ case X86::BI__builtin_ia32_insertf32x4:
+ case X86::BI__builtin_ia32_inserti64x4:
+ case X86::BI__builtin_ia32_inserti32x4:
+ case X86::BI__builtin_ia32_insertf32x8:
+ case X86::BI__builtin_ia32_inserti32x8:
+ case X86::BI__builtin_ia32_insertf32x4_256:
+ case X86::BI__builtin_ia32_inserti32x4_256:
+ case X86::BI__builtin_ia32_insertf64x2_256:
+ case X86::BI__builtin_ia32_inserti64x2_256:
+ case X86::BI__builtin_ia32_insertf64x2_512:
+ case X86::BI__builtin_ia32_inserti64x2_512:
+ case X86::BI__builtin_ia32_pmovqd512_mask:
+ case X86::BI__builtin_ia32_pmovwb512_mask:
+ case X86::BI__builtin_ia32_pblendw128:
+ case X86::BI__builtin_ia32_blendpd:
+ case X86::BI__builtin_ia32_blendps:
+ case X86::BI__builtin_ia32_blendpd256:
+ case X86::BI__builtin_ia32_blendps256:
+ case X86::BI__builtin_ia32_pblendw256:
+ case X86::BI__builtin_ia32_pblendd128:
+ case X86::BI__builtin_ia32_pblendd256:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_pshuflw:
+ case X86::BI__builtin_ia32_pshuflw256:
+ case X86::BI__builtin_ia32_pshuflw512:
+ return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
+ true);
+ case X86::BI__builtin_ia32_pshufhw:
+ case X86::BI__builtin_ia32_pshufhw256:
+ case X86::BI__builtin_ia32_pshufhw512:
+ return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
+ false);
+ case X86::BI__builtin_ia32_pshufd:
+ case X86::BI__builtin_ia32_pshufd256:
+ case X86::BI__builtin_ia32_pshufd512:
+ case X86::BI__builtin_ia32_vpermilpd:
+ case X86::BI__builtin_ia32_vpermilps:
+ case X86::BI__builtin_ia32_vpermilpd256:
+ case X86::BI__builtin_ia32_vpermilps256:
+ case X86::BI__builtin_ia32_vpermilpd512:
+ case X86::BI__builtin_ia32_vpermilps512: {
+ const uint32_t imm = getSExtIntValueFromConstOp(ops[1]);
+
+ llvm::SmallVector<int64_t, 16> mask(16);
+ computeFullLaneShuffleMask(*this, ops[0], imm, false, mask);
+
+ return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], mask);
+ }
+ case X86::BI__builtin_ia32_shufpd:
+ case X86::BI__builtin_ia32_shufpd256:
+ case X86::BI__builtin_ia32_shufpd512:
+ case X86::BI__builtin_ia32_shufps:
+ case X86::BI__builtin_ia32_shufps256:
+ case X86::BI__builtin_ia32_shufps512: {
+ const uint32_t imm = getZExtIntValueFromConstOp(ops[2]);
+
+ llvm::SmallVector<int64_t, 16> mask(16);
+ computeFullLaneShuffleMask(*this, ops[0], imm, true, mask);
+
+ return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0],
+ ops[1], mask);
+ }
+ case X86::BI__builtin_ia32_permdi256:
+ case X86::BI__builtin_ia32_permdf256:
+ case X86::BI__builtin_ia32_permdi512:
+ case X86::BI__builtin_ia32_permdf512:
+ case X86::BI__builtin_ia32_palignr128:
+ case X86::BI__builtin_ia32_palignr256:
+ case X86::BI__builtin_ia32_palignr512:
+ case X86::BI__builtin_ia32_alignd128:
+ case X86::BI__builtin_ia32_alignd256:
+ case X86::BI__builtin_ia32_alignd512:
+ case X86::BI__builtin_ia32_alignq128:
+ case X86::BI__builtin_ia32_alignq256:
+ case X86::BI__builtin_ia32_alignq512:
+ case X86::BI__builtin_ia32_shuf_f32x4_256:
+ case X86::BI__builtin_ia32_shuf_f64x2_256:
+ case X86::BI__builtin_ia32_shuf_i32x4_256:
+ case X86::BI__builtin_ia32_shuf_i64x2_256:
+ case X86::BI__builtin_ia32_shuf_f32x4:
+ case X86::BI__builtin_ia32_shuf_f64x2:
+ case X86::BI__builtin_ia32_shuf_i32x4:
+ case X86::BI__builtin_ia32_shuf_i64x2:
+ case X86::BI__builtin_ia32_vperm2f128_pd256:
+ case X86::BI__builtin_ia32_vperm2f128_ps256:
+ case X86::BI__builtin_ia32_vperm2f128_si256:
+ case X86::BI__builtin_ia32_permti256:
+ case X86::BI__builtin_ia32_pslldqi128_byteshift:
+ case X86::BI__builtin_ia32_pslldqi256_byteshift:
+ case X86::BI__builtin_ia32_pslldqi512_byteshift:
+ case X86::BI__builtin_ia32_psrldqi128_byteshift:
+ case X86::BI__builtin_ia32_psrldqi256_byteshift:
+ case X86::BI__builtin_ia32_psrldqi512_byteshift:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_kshiftliqi:
+ case X86::BI__builtin_ia32_kshiftlihi:
+ case X86::BI__builtin_ia32_kshiftlisi:
+ case X86::BI__builtin_ia32_kshiftlidi: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ unsigned shiftVal =
+ ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
+ 0xff;
+ unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
+
+ if (shiftVal >= numElems)
+ return builder.getNullValue(ops[0].getType(), loc);
+
+ mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
+
+ SmallVector<mlir::Attribute, 64> indices;
+ mlir::Type i32Ty = builder.getSInt32Ty();
+ for (auto i : llvm::seq<unsigned>(0, numElems))
+ indices.push_back(cir::IntAttr::get(i32Ty, numElems + i - shiftVal));
+
+ mlir::Value zero = builder.getNullValue(in.getType(), loc);
+ mlir::Value sv = builder.createVecShuffle(loc, zero, in, indices);
+ return builder.createBitcast(sv, ops[0].getType());
+ }
+ case X86::BI__builtin_ia32_kshiftriqi:
+ case X86::BI__builtin_ia32_kshiftrihi:
+ case X86::BI__builtin_ia32_kshiftrisi:
+ case X86::BI__builtin_ia32_kshiftridi: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ unsigned shiftVal =
+ ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
+ 0xff;
+ unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
+
+ if (shiftVal >= numElems)
+ return builder.getNullValue(ops[0].getType(), loc);
+
+ mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
+
+ SmallVector<mlir::Attribute, 64> indices;
+ mlir::Type i32Ty = builder.getSInt32Ty();
+ for (auto i : llvm::seq<unsigned>(0, numElems))
+ indices.push_back(cir::IntAttr::get(i32Ty, i + shiftVal));
+
+ mlir::Value zero = builder.getNullValue(in.getType(), loc);
+ mlir::Value sv = builder.createVecShuffle(loc, in, zero, indices);
+ return builder.createBitcast(sv, ops[0].getType());
+ }
+ case X86::BI__builtin_ia32_vprotbi:
+ case X86::BI__builtin_ia32_vprotwi:
+ case X86::BI__builtin_ia32_vprotdi:
+ case X86::BI__builtin_ia32_vprotqi:
+ case X86::BI__builtin_ia32_prold128:
+ case X86::BI__builtin_ia32_prold256:
+ case X86::BI__builtin_ia32_prold512:
+ case X86::BI__builtin_ia32_prolq128:
+ case X86::BI__builtin_ia32_prolq256:
+ case X86::BI__builtin_ia32_prolq512:
+ return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
+ ops[0], ops[1], false);
+ case X86::BI__builtin_ia32_prord128:
+ case X86::BI__builtin_ia32_prord256:
+ case X86::BI__builtin_ia32_prord512:
+ case X86::BI__builtin_ia32_prorq128:
+ case X86::BI__builtin_ia32_prorq256:
+ case X86::BI__builtin_ia32_prorq512:
+ return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
+ ops[0], ops[1], true);
+ case X86::BI__builtin_ia32_selectb_128:
+ case X86::BI__builtin_ia32_selectb_256:
+ case X86::BI__builtin_ia32_selectb_512:
+ case X86::BI__builtin_ia32_selectw_128:
+ case X86::BI__builtin_ia32_selectw_256:
+ case X86::BI__builtin_ia32_selectw_512:
+ case X86::BI__builtin_ia32_selectd_128:
+ case X86::BI__builtin_ia32_selectd_256:
+ case X86::BI__builtin_ia32_selectd_512:
+ case X86::BI__builtin_ia32_selectq_128:
+ case X86::BI__builtin_ia32_selectq_256:
+ case X86::BI__builtin_ia32_selectq_512:
+ case X86::BI__builtin_ia32_selectph_128:
+ case X86::BI__builtin_ia32_selectph_256:
+ case X86::BI__builtin_ia32_selectph_512:
+ case X86::BI__builtin_ia32_selectpbf_128:
+ case X86::BI__builtin_ia32_selectpbf_256:
+ case X86::BI__builtin_ia32_selectpbf_512:
+ case X86::BI__builtin_ia32_selectps_128:
+ case X86::BI__builtin_ia32_selectps_256:
+ case X86::BI__builtin_ia32_selectps_512:
+ case X86::BI__builtin_ia32_selectpd_128:
+ case X86::BI__builtin_ia32_selectpd_256:
+ case X86::BI__builtin_ia32_selectpd_512:
+ return emitX86Select(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
+ ops[2]);
+ case X86::BI__builtin_ia32_selectsh_128:
+ case X86::BI__builtin_ia32_selectsbf_128:
+ case X86::BI__builtin_ia32_selectss_128:
+ case X86::BI__builtin_ia32_selectsd_128:
+ return emitX86ScalarSelect(builder, getLoc(expr->getExprLoc()), ops[0],
+ ops[1], ops[2]);
+ case X86::BI__builtin_ia32_cmpb128_mask:
+ case X86::BI__builtin_ia32_cmpb256_mask:
+ case X86::BI__builtin_ia32_cmpb512_mask:
+ case X86::BI__builtin_ia32_cmpw128_mask:
+ case X86::BI__builtin_ia32_cmpw256_mask:
+ case X86::BI__builtin_ia32_cmpw512_mask:
+ case X86::BI__builtin_ia32_cmpd128_mask:
+ case X86::BI__builtin_ia32_cmpd256_mask:
+ case X86::BI__builtin_ia32_cmpd512_mask:
+ case X86::BI__builtin_ia32_cmpq128_mask:
+ case X86::BI__builtin_ia32_cmpq256_mask:
+ case X86::BI__builtin_ia32_cmpq512_mask:
+ case X86::BI__builtin_ia32_ucmpb128_mask:
+ case X86::BI__builtin_ia32_ucmpb256_mask:
+ case X86::BI__builtin_ia32_ucmpb512_mask:
+ case X86::BI__builtin_ia32_ucmpw128_mask:
+ case X86::BI__builtin_ia32_ucmpw256_mask:
+ case X86::BI__builtin_ia32_ucmpw512_mask:
+ case X86::BI__builtin_ia32_ucmpd128_mask:
+ case X86::BI__builtin_ia32_ucmpd256_mask:
+ case X86::BI__builtin_ia32_ucmpd512_mask:
+ case X86::BI__builtin_ia32_ucmpq128_mask:
+ case X86::BI__builtin_ia32_ucmpq256_mask:
+ case X86::BI__builtin_ia32_ucmpq512_mask:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_vpcomb:
+ case X86::BI__builtin_ia32_vpcomw:
+ case X86::BI__builtin_ia32_vpcomd:
+ case X86::BI__builtin_ia32_vpcomq:
+ return emitX86vpcom(builder, getLoc(expr->getExprLoc()), ops, true);
+ case X86::BI__builtin_ia32_vpcomub:
+ case X86::BI__builtin_ia32_vpcomuw:
+ case X86::BI__builtin_ia32_vpcomud:
+ case X86::BI__builtin_ia32_vpcomuq:
+ return emitX86vpcom(builder, getLoc(expr->getExprLoc()), ops, false);
+ case X86::BI__builtin_ia32_kortestcqi:
+ case X86::BI__builtin_ia32_kortestchi:
+ case X86::BI__builtin_ia32_kortestcsi:
+ case X86::BI__builtin_ia32_kortestcdi: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ cir::IntType ty = cast<cir::IntType>(ops[0].getType());
+ mlir::Value allOnesOp =
+ builder.getConstAPInt(loc, ty, APInt::getAllOnes(ty.getWidth()));
+ mlir::Value orOp =
+ emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
+ mlir::Value cmp =
+ cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allOnesOp);
+ return builder.createCast(cir::CastKind::bool_to_int, cmp,
+ cgm.convertType(expr->getType()));
+ }
+ case X86::BI__builtin_ia32_kortestzqi:
+ case X86::BI__builtin_ia32_kortestzhi:
+ case X86::BI__builtin_ia32_kortestzsi:
+ case X86::BI__builtin_ia32_kortestzdi: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ cir::IntType ty = cast<cir::IntType>(ops[0].getType());
+ mlir::Value allZerosOp = builder.getNullValue(ty, loc).getResult();
+ mlir::Value orOp =
+ emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
+ mlir::Value cmp = cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq,
+ orOp, allZerosOp);
+ return builder.createCast(cir::CastKind::bool_to_int, cmp,
+ cgm.convertType(expr->getType()));
+ }
+ case X86::BI__builtin_ia32_ktestcqi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestc.b", ops);
+ case X86::BI__builtin_ia32_ktestzqi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestz.b", ops);
+ case X86::BI__builtin_ia32_ktestchi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestc.w", ops);
+ case X86::BI__builtin_ia32_ktestzhi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestz.w", ops);
+ case X86::BI__builtin_ia32_ktestcsi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestc.d", ops);
+ case X86::BI__builtin_ia32_ktestzsi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestz.d", ops);
+ case X86::BI__builtin_ia32_ktestcdi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestc.q", ops);
+ case X86::BI__builtin_ia32_ktestzdi:
+ return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.ktestz.q", ops);
+ case X86::BI__builtin_ia32_kaddqi:
+ return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kadd.b", ops);
+ case X86::BI__builtin_ia32_kaddhi:
+ return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kadd.w", ops);
+ case X86::BI__builtin_ia32_kaddsi:
+ return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kadd.d", ops);
+ case X86::BI__builtin_ia32_kadddi:
+ return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
+ "x86.avx512.kadd.q", ops);
+ case X86::BI__builtin_ia32_kandqi:
+ case X86::BI__builtin_ia32_kandhi:
+ case X86::BI__builtin_ia32_kandsi:
+ case X86::BI__builtin_ia32_kanddi:
+ return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
+ cir::BinOpKind::And, ops);
+ case X86::BI__builtin_ia32_kandnqi:
+ case X86::BI__builtin_ia32_kandnhi:
+ case X86::BI__builtin_ia32_kandnsi:
+ case X86::BI__builtin_ia32_kandndi:
+ return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
+ cir::BinOpKind::And, ops, true);
+ case X86::BI__builtin_ia32_korqi:
+ case X86::BI__builtin_ia32_korhi:
+ case X86::BI__builtin_ia32_korsi:
+ case X86::BI__builtin_ia32_kordi:
+ return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
+ cir::BinOpKind::Or, ops);
+ case X86::BI__builtin_ia32_kxnorqi:
+ case X86::BI__builtin_ia32_kxnorhi:
+ case X86::BI__builtin_ia32_kxnorsi:
+ case X86::BI__builtin_ia32_kxnordi:
+ return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
+ cir::BinOpKind::Xor, ops, true);
+ case X86::BI__builtin_ia32_kxorqi:
+ case X86::BI__builtin_ia32_kxorhi:
+ case X86::BI__builtin_ia32_kxorsi:
+ case X86::BI__builtin_ia32_kxordi:
+ return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
+ cir::BinOpKind::Xor, ops);
+ case X86::BI__builtin_ia32_knotqi:
+ case X86::BI__builtin_ia32_knothi:
+ case X86::BI__builtin_ia32_knotsi:
+ case X86::BI__builtin_ia32_knotdi: {
+ cir::IntType intTy = cast<cir::IntType>(ops[0].getType());
+ unsigned numElts = intTy.getWidth();
+ mlir::Value resVec =
+ getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
+ return builder.createBitcast(builder.createNot(resVec), ops[0].getType());
+ }
+ case X86::BI__builtin_ia32_kmovb:
+ case X86::BI__builtin_ia32_kmovw:
+ case X86::BI__builtin_ia32_kmovd:
+ case X86::BI__builtin_ia32_kmovq: {
+ // Bitcast to vXi1 type and then back to integer. This gets the mask
+ // register type into the IR, but might be optimized out depending on
+ // what's around it.
+ cir::IntType intTy = cast<cir::IntType>(ops[0].getType());
+ unsigned numElts = intTy.getWidth();
+ mlir::Value resVec =
+ getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
+ return builder.createBitcast(resVec, ops[0].getType());
+ }
+ case X86::BI__builtin_ia32_sqrtsh_round_mask:
+ case X86::BI__builtin_ia32_sqrtsd_round_mask:
+ case X86::BI__builtin_ia32_sqrtss_round_mask:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_sqrtph512:
+ case X86::BI__builtin_ia32_sqrtps512:
+ case X86::BI__builtin_ia32_sqrtpd512: {
+ mlir::Location loc = getLoc(expr->getExprLoc());
+ mlir::Value arg = ops[0];
+ return cir::SqrtOp::create(builder, loc, arg.getType(), arg).getResult();
+ }
+ case X86::BI__builtin_ia32_pmuludq128:
+ case X86::BI__builtin_ia32_pmuludq256:
+ case X86::BI__builtin_ia32_pmuludq512: {
+ unsigned opTypePrimitiveSizeInBits =
+ cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
+ return emitX86Muldq(builder, getLoc(expr->getExprLoc()),
+ /*isSigned*/ false, ops, opTypePrimitiveSizeInBits);
+ }
+ case X86::BI__builtin_ia32_pmuldq128:
+ case X86::BI__builtin_ia32_pmuldq256:
+ case X86::BI__builtin_ia32_pmuldq512: {
+ unsigned opTypePrimitiveSizeInBits =
+ cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
+ return emitX86Muldq(builder, getLoc(expr->getExprLoc()),
+ /*isSigned*/ true, ops, opTypePrimitiveSizeInBits);
+ }
+ case X86::BI__builtin_ia32_pternlogd512_mask:
+ case X86::BI__builtin_ia32_pternlogq512_mask:
+ case X86::BI__builtin_ia32_pternlogd128_mask:
+ case X86::BI__builtin_ia32_pternlogd256_mask:
+ case X86::BI__builtin_ia32_pternlogq128_mask:
+ case X86::BI__builtin_ia32_pternlogq256_mask:
+ case X86::BI__builtin_ia32_pternlogd512_maskz:
+ case X86::BI__builtin_ia32_pternlogq512_maskz:
+ case X86::BI__builtin_ia32_pternlogd128_maskz:
+ case X86::BI__builtin_ia32_pternlogd256_maskz:
+ case X86::BI__builtin_ia32_pternlogq128_maskz:
+ case X86::BI__builtin_ia32_pternlogq256_maskz:
+ case X86::BI__builtin_ia32_vpshldd128:
+ case X86::BI__builtin_ia32_vpshldd256:
+ case X86::BI__builtin_ia32_vpshldd512:
+ case X86::BI__builtin_ia32_vpshldq128:
+ case X86::BI__builtin_ia32_vpshldq256:
+ case X86::BI__builtin_ia32_vpshldq512:
+ case X86::BI__builtin_ia32_vpshldw128:
+ case X86::BI__builtin_ia32_vpshldw256:
+ case X86::BI__builtin_ia32_vpshldw512:
+ case X86::BI__builtin_ia32_vpshrdd128:
+ case X86::BI__builtin_ia32_vpshrdd256:
+ case X86::BI__builtin_ia32_vpshrdd512:
+ case X86::BI__builtin_ia32_vpshrdq128:
+ case X86::BI__builtin_ia32_vpshrdq256:
+ case X86::BI__builtin_ia32_vpshrdq512:
+ case X86::BI__builtin_ia32_vpshrdw128:
+ case X86::BI__builtin_ia32_vpshrdw256:
+ case X86::BI__builtin_ia32_vpshrdw512:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return {};
+ case X86::BI__builtin_ia32_reduce_fadd_pd512:
+ case X86::BI__builtin_ia32_reduce_fadd_ps512:
+ case X86::BI__builtin_ia32_reduce_fadd_ph512:
+ case X86::BI__builtin_ia32_reduce_fadd_ph256:
+ case X86::BI__builtin_ia32_reduce_fadd_ph128: {
+ assert(!cir::MissingFeatures::fastMathFlags());
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "vector.reduce.fadd", ops[0].getType(),
+ mlir::ValueRange{ops[0], ops[1]});
+ }
+ case X86::BI__builtin_ia32_reduce_fmul_pd512:
+ case X86::BI__builtin_ia32_reduce_fmul_ps512:
+ case X86::BI__builtin_ia32_reduce_fmul_ph512:
+ case X86::BI__builtin_ia32_reduce_fmul_ph256:
+ case X86::BI__builtin_ia32_reduce_fmul_ph128: {
+ assert(!cir::MissingFeatures::fastMathFlags());
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "vector.reduce.fmul", ops[0].getType(),
+ mlir::ValueRange{ops[0], ops[1]});
+ }
+ case X86::BI__builtin_ia32_reduce_fmax_pd512:
+ case X86::BI__builtin_ia32_reduce_fmax_ps512:
+ case X86::BI__builtin_ia32_reduce_fmax_ph512:
+ case X86::BI__builtin_ia32_reduce_fmax_ph256:
+ case X86::BI__builtin_ia32_reduce_fmax_ph128: {
+ assert(!cir::MissingFeatures::fastMathFlags());
+ cir::VectorType vecTy = cast<cir::VectorType>(ops[0].getType());
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "vector.reduce.fmax", vecTy.getElementType(),
+ mlir::ValueRange{ops[0]});
+ }
+ case X86::BI__builtin_ia32_reduce_fmin_pd512:
+ case X86::BI__builtin_ia32_reduce_fmin_ps512:
+ case X86::BI__builtin_ia32_reduce_fmin_ph512:
+ case X86::BI__builtin_ia32_reduce_fmin_ph256:
+ case X86::BI__builtin_ia32_reduce_fmin_ph128: {
+ assert(!cir::MissingFeatures::fastMathFlags());
+ cir::VectorType vecTy = cast<cir::VectorType>(ops[0].getType());
+ return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
+ "vector.reduce.fmin", vecTy.getElementType(),
+ mlir::ValueRange{ops[0]});
+ }
+ case X86::BI__builtin_ia32_rdrand16_step:
+ case X86::BI__builtin_ia32_rdrand32_step:
+ case X86::BI__builtin_ia32_rdrand64_step:
+ case X86::BI__builtin_ia32_rdseed16_step:
+ case X86::BI__builtin_ia32_rdseed32_step:
+ case X86::BI__builtin_ia32_rdseed64_step:
+ case X86::BI__builtin_ia32_addcarryx_u32:
+ case X86::BI__builtin_ia32_addcarryx_u64:
+ case X86::BI__builtin_ia32_subborrow_u32:
+ case X86::BI__builtin_ia32_subborrow_u64:
+ case X86::BI__builtin_ia32_fpclassps128_mask:
+ case X86::BI__builtin_ia32_fpclassps256_mask:
+ case X86::BI__builtin_ia32_fpclassps512_mask:
+ case X86::BI__builtin_ia32_vfpclassbf16128_mask:
+ case X86::BI__builtin_ia32_vfpclassbf16256_mask:
+ case X86::BI__builtin_ia32_vfpclassbf16512_mask:
+ case X86::BI__builtin_ia32_fpclassph128_mask:
+ case X86::BI__builtin_ia32_fpclassph256_mask:
+ case X86::BI__builtin_ia32_fpclassph512_mask:
+ case X86::BI__builtin_ia32_fpclasspd128_mask:
+ case X86::BI__builtin_ia32_fpclasspd256_mask:
+ case X86::BI__builtin_ia32_fpclasspd512_mask:
+ case X86::BI__builtin_ia32_vp2intersect_q_512:
+ case X86::BI__builtin_ia32_vp2intersect_q_256:
+ case X86::BI__builtin_ia32_vp2intersect_q_128:
+ case X86::BI__builtin_ia32_vp2intersect_d_512:
+ case X86::BI__builtin_ia32_vp2intersect_d_256:
+ case X86::BI__builtin_ia32_vp2intersect_d_128:
+ case X86::BI__builtin_ia32_vpmultishiftqb128:
+ case X86::BI__builtin_ia32_vpmultishiftqb256:
+ case X86::BI__builtin_ia32_vpmultishiftqb512:
+ case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
+ case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
+ case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
+ case X86::BI__builtin_ia32_cmpeqps:
+ case X86::BI__builtin_ia32_cmpeqpd:
+ case X86::BI__builtin_ia32_cmpltps:
+ case X86::BI__builtin_ia32_cmpltpd:
+ case X86::BI__builtin_ia32_cmpleps:
+ case X86::BI__builtin_ia32_cmplepd:
+ case X86::BI__builtin_ia32_cmpunordps:
+ case X86::BI__builtin_ia32_cmpunordpd:
+ case X86::BI__builtin_ia32_cmpneqps:
+ case X86::BI__builtin_ia32_cmpneqpd:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ case X86::BI__builtin_ia32_cmpnltps:
+ case X86::BI__builtin_ia32_cmpnltpd:
+ return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),
+ cir::CmpOpKind::lt, /*shouldInvert=*/true);
+ case X86::BI__builtin_ia32_cmpnleps:
+ case X86::BI__builtin_ia32_cmpnlepd:
+ return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),
+ cir::CmpOpKind::le, /*shouldInvert=*/true);
+ case X86::BI__builtin_ia32_cmpordps:
+ case X86::BI__builtin_ia32_cmpordpd:
+ case X86::BI__builtin_ia32_cmpph128_mask:
+ case X86::BI__builtin_ia32_cmpph256_mask:
+ case X86::BI__builtin_ia32_cmpph512_mask:
+ case X86::BI__builtin_ia32_cmpps128_mask:
+ case X86::BI__builtin_ia32_cmpps256_mask:
+ case X86::BI__builtin_ia32_cmpps512_mask:
+ case X86::BI__builtin_ia32_cmppd128_mask:
+ case X86::BI__builtin_ia32_cmppd256_mask:
+ case X86::BI__builtin_ia32_cmppd512_mask:
+ case X86::BI__builtin_ia32_vcmpbf16512_mask:
+ case X86::BI__builtin_ia32_vcmpbf16256_mask:
+ case X86::BI__builtin_ia32_vcmpbf16128_mask:
+ case X86::BI__builtin_ia32_cmpps:
+ case X86::BI__builtin_ia32_cmpps256:
+ case X86::BI__builtin_ia32_cmppd:
+ case X86::BI__builtin_ia32_cmppd256:
+ case X86::BI__builtin_ia32_cmpeqss:
+ case X86::BI__builtin_ia32_cmpltss:
+ case X86::BI__builtin_ia32_cmpless:
+ case X86::BI__builtin_ia32_cmpunordss:
+ case X86::BI__builtin_ia32_cmpneqss:
+ case X86::BI__builtin_ia32_cmpnltss:
+ case X86::BI__builtin_ia32_cmpnless:
+ case X86::BI__builtin_ia32_cmpordss:
+ case X86::BI__builtin_ia32_cmpeqsd:
+ case X86::BI__builtin_ia32_cmpltsd:
+ case X86::BI__builtin_ia32_cmplesd:
+ case X86::BI__builtin_ia32_cmpunordsd:
+ case X86::BI__builtin_ia32_cmpneqsd:
+ case X86::BI__builtin_ia32_cmpnltsd:
+ case X86::BI__builtin_ia32_cmpnlesd:
+ case X86::BI__builtin_ia32_cmpordsd:
+ case X86::BI__builtin_ia32_vcvtph2ps_mask:
+ case X86::BI__builtin_ia32_vcvtph2ps256_mask:
+ case X86::BI__builtin_ia32_vcvtph2ps512_mask:
+ case X86::BI__builtin_ia32_cvtneps2bf16_128_mask:
+ case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
+ case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
+ case X86::BI__cpuid:
+ case X86::BI__cpuidex:
+ case X86::BI__emul:
+ case X86::BI__emulu:
+ case X86::BI__mulh:
+ case X86::BI__umulh:
+ case X86::BI_mul128:
+ case X86::BI_umul128: {
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ }
+ case X86::BI__faststorefence: {
+ cir::AtomicFenceOp::create(
+ builder, getLoc(expr->getExprLoc()),
+ cir::MemOrder::SequentiallyConsistent,
+ cir::SyncScopeKindAttr::get(&getMLIRContext(),
+ cir::SyncScopeKind::System));
+ return mlir::Value{};
+ }
+ case X86::BI__shiftleft128:
+ case X86::BI__shiftright128: {
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
+ }
+ case X86::BI_ReadWriteBarrier:
+ case X86::BI_ReadBarrier:
+ case X86::BI_WriteBarrier: {
+ cir::AtomicFenceOp::create(
+ builder, getLoc(expr->getExprLoc()),
+ cir::MemOrder::SequentiallyConsistent,
+ cir::SyncScopeKindAttr::get(&getMLIRContext(),
+ cir::SyncScopeKind::SingleThread));
+ return mlir::Value{};
+ }
+ case X86::BI_AddressOfReturnAddress:
+ case X86::BI__stosb:
+ case X86::BI__ud2:
+ case X86::BI__int2c:
+ case X86::BI__readfsbyte:
+ case X86::BI__readfsword:
+ case X86::BI__readfsdword:
+ case X86::BI__readfsqword:
+ case X86::BI__readgsbyte:
+ case X86::BI__readgsword:
+ case X86::BI__readgsdword:
+ case X86::BI__readgsqword:
+ case X86::BI__builtin_ia32_encodekey128_u32:
+ case X86::BI__builtin_ia32_encodekey256_u32:
+ case X86::BI__builtin_ia32_aesenc128kl_u8:
+ case X86::BI__builtin_ia32_aesdec128kl_u8:
+ case X86::BI__builtin_ia32_aesenc256kl_u8:
+ case X86::BI__builtin_ia32_aesdec256kl_u8:
+ case X86::BI__builtin_ia32_aesencwide128kl_u8:
+ case X86::BI__builtin_ia32_aesdecwide128kl_u8:
+ case X86::BI__builtin_ia32_aesencwide256kl_u8:
+ case X86::BI__builtin_ia32_aesdecwide256kl_u8:
+ case X86::BI__builtin_ia32_vfcmaddcph512_mask:
+ case X86::BI__builtin_ia32_vfmaddcph512_mask:
+ case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
+ case X86::BI__builtin_ia32_vfmaddcsh_round_mask:
+ case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
+ case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
+ case X86::BI__builtin_ia32_prefetchi:
+ cgm.errorNYI(expr->getSourceRange(),
+ std::string("unimplemented X86 builtin call: ") +
+ getContext().BuiltinInfo.getName(builtinID));
+ return mlir::Value{};
}
-
- mlir::Location loc = getLoc(expr->getExprLoc());
- unsigned minElts =
- std::min(cast<cir::VectorType>(ops[2].getType()).getSize(),
- cast<cir::VectorType>(ops[3].getType()).getSize());
- ops[1] = getMaskVecValue(builder, loc, ops[1], minElts);
-
- return emitIntrinsicCallOp(builder, loc, intrinsicName,
- convertType(expr->getType()), ops);
- }
- case X86::BI__builtin_ia32_vextractf128_pd256:
- case X86::BI__builtin_ia32_vextractf128_ps256:
- case X86::BI__builtin_ia32_vextractf128_si256:
- case X86::BI__builtin_ia32_extract128i256:
- case X86::BI__builtin_ia32_extractf64x4_mask:
- case X86::BI__builtin_ia32_extractf32x4_mask:
- case X86::BI__builtin_ia32_extracti64x4_mask:
- case X86::BI__builtin_ia32_extracti32x4_mask:
- case X86::BI__builtin_ia32_extractf32x8_mask:
- case X86::BI__builtin_ia32_extracti32x8_mask:
- case X86::BI__builtin_ia32_extractf32x4_256_mask:
- case X86::BI__builtin_ia32_extracti32x4_256_mask:
- case X86::BI__builtin_ia32_extractf64x2_256_mask:
- case X86::BI__builtin_ia32_extracti64x2_256_mask:
- case X86::BI__builtin_ia32_extractf64x2_512_mask:
- case X86::BI__builtin_ia32_extracti64x2_512_mask: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- cir::VectorType dstTy = cast<cir::VectorType>(convertType(expr->getType()));
- unsigned numElts = dstTy.getSize();
- unsigned srcNumElts = cast<cir::VectorType>(ops[0].getType()).getSize();
- unsigned subVectors = srcNumElts / numElts;
- assert(llvm::isPowerOf2_32(subVectors) && "Expected power of 2 subvectors");
- unsigned index =
- ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
-
- index &= subVectors - 1; // Remove any extra bits.
- index *= numElts;
-
- int64_t indices[16];
- std::iota(indices, indices + numElts, index);
-
- mlir::Value poison =
- builder.getConstant(loc, cir::PoisonAttr::get(ops[0].getType()));
- mlir::Value res = builder.createVecShuffle(loc, ops[0], poison,
- ArrayRef(indices, numElts));
- if (ops.size() == 4)
- res = emitX86Select(builder, loc, ops[3], res, ops[2]);
-
- return res;
- }
- case X86::BI__builtin_ia32_vinsertf128_pd256:
- case X86::BI__builtin_ia32_vinsertf128_ps256:
- case X86::BI__builtin_ia32_vinsertf128_si256:
- case X86::BI__builtin_ia32_insert128i256:
- case X86::BI__builtin_ia32_insertf64x4:
- case X86::BI__builtin_ia32_insertf32x4:
- case X86::BI__builtin_ia32_inserti64x4:
- case X86::BI__builtin_ia32_inserti32x4:
- case X86::BI__builtin_ia32_insertf32x8:
- case X86::BI__builtin_ia32_inserti32x8:
- case X86::BI__builtin_ia32_insertf32x4_256:
- case X86::BI__builtin_ia32_inserti32x4_256:
- case X86::BI__builtin_ia32_insertf64x2_256:
- case X86::BI__builtin_ia32_inserti64x2_256:
- case X86::BI__builtin_ia32_insertf64x2_512:
- case X86::BI__builtin_ia32_inserti64x2_512:
- case X86::BI__builtin_ia32_pmovqd512_mask:
- case X86::BI__builtin_ia32_pmovwb512_mask:
- case X86::BI__builtin_ia32_pblendw128:
- case X86::BI__builtin_ia32_blendpd:
- case X86::BI__builtin_ia32_blendps:
- case X86::BI__builtin_ia32_blendpd256:
- case X86::BI__builtin_ia32_blendps256:
- case X86::BI__builtin_ia32_pblendw256:
- case X86::BI__builtin_ia32_pblendd128:
- case X86::BI__builtin_ia32_pblendd256:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_pshuflw:
- case X86::BI__builtin_ia32_pshuflw256:
- case X86::BI__builtin_ia32_pshuflw512:
- return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
- true);
- case X86::BI__builtin_ia32_pshufhw:
- case X86::BI__builtin_ia32_pshufhw256:
- case X86::BI__builtin_ia32_pshufhw512:
- return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
- false);
- case X86::BI__builtin_ia32_pshufd:
- case X86::BI__builtin_ia32_pshufd256:
- case X86::BI__builtin_ia32_pshufd512:
- case X86::BI__builtin_ia32_vpermilpd:
- case X86::BI__builtin_ia32_vpermilps:
- case X86::BI__builtin_ia32_vpermilpd256:
- case X86::BI__builtin_ia32_vpermilps256:
- case X86::BI__builtin_ia32_vpermilpd512:
- case X86::BI__builtin_ia32_vpermilps512: {
- const uint32_t imm = getSExtIntValueFromConstOp(ops[1]);
-
- llvm::SmallVector<int64_t, 16> mask(16);
- computeFullLaneShuffleMask(*this, ops[0], imm, false, mask);
-
- return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], mask);
- }
- case X86::BI__builtin_ia32_shufpd:
- case X86::BI__builtin_ia32_shufpd256:
- case X86::BI__builtin_ia32_shufpd512:
- case X86::BI__builtin_ia32_shufps:
- case X86::BI__builtin_ia32_shufps256:
- case X86::BI__builtin_ia32_shufps512: {
- const uint32_t imm = getZExtIntValueFromConstOp(ops[2]);
-
- llvm::SmallVector<int64_t, 16> mask(16);
- computeFullLaneShuffleMask(*this, ops[0], imm, true, mask);
-
- return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], ops[1],
- mask);
- }
- case X86::BI__builtin_ia32_permdi256:
- case X86::BI__builtin_ia32_permdf256:
- case X86::BI__builtin_ia32_permdi512:
- case X86::BI__builtin_ia32_permdf512:
- case X86::BI__builtin_ia32_palignr128:
- case X86::BI__builtin_ia32_palignr256:
- case X86::BI__builtin_ia32_palignr512:
- case X86::BI__builtin_ia32_alignd128:
- case X86::BI__builtin_ia32_alignd256:
- case X86::BI__builtin_ia32_alignd512:
- case X86::BI__builtin_ia32_alignq128:
- case X86::BI__builtin_ia32_alignq256:
- case X86::BI__builtin_ia32_alignq512:
- case X86::BI__builtin_ia32_shuf_f32x4_256:
- case X86::BI__builtin_ia32_shuf_f64x2_256:
- case X86::BI__builtin_ia32_shuf_i32x4_256:
- case X86::BI__builtin_ia32_shuf_i64x2_256:
- case X86::BI__builtin_ia32_shuf_f32x4:
- case X86::BI__builtin_ia32_shuf_f64x2:
- case X86::BI__builtin_ia32_shuf_i32x4:
- case X86::BI__builtin_ia32_shuf_i64x2:
- case X86::BI__builtin_ia32_vperm2f128_pd256:
- case X86::BI__builtin_ia32_vperm2f128_ps256:
- case X86::BI__builtin_ia32_vperm2f128_si256:
- case X86::BI__builtin_ia32_permti256:
- case X86::BI__builtin_ia32_pslldqi128_byteshift:
- case X86::BI__builtin_ia32_pslldqi256_byteshift:
- case X86::BI__builtin_ia32_pslldqi512_byteshift:
- case X86::BI__builtin_ia32_psrldqi128_byteshift:
- case X86::BI__builtin_ia32_psrldqi256_byteshift:
- case X86::BI__builtin_ia32_psrldqi512_byteshift:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_kshiftliqi:
- case X86::BI__builtin_ia32_kshiftlihi:
- case X86::BI__builtin_ia32_kshiftlisi:
- case X86::BI__builtin_ia32_kshiftlidi: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- unsigned shiftVal =
- ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
- 0xff;
- unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
-
- if (shiftVal >= numElems)
- return builder.getNullValue(ops[0].getType(), loc);
-
- mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
-
- SmallVector<mlir::Attribute, 64> indices;
- mlir::Type i32Ty = builder.getSInt32Ty();
- for (auto i : llvm::seq<unsigned>(0, numElems))
- indices.push_back(cir::IntAttr::get(i32Ty, numElems + i - shiftVal));
-
- mlir::Value zero = builder.getNullValue(in.getType(), loc);
- mlir::Value sv = builder.createVecShuffle(loc, zero, in, indices);
- return builder.createBitcast(sv, ops[0].getType());
- }
- case X86::BI__builtin_ia32_kshiftriqi:
- case X86::BI__builtin_ia32_kshiftrihi:
- case X86::BI__builtin_ia32_kshiftrisi:
- case X86::BI__builtin_ia32_kshiftridi: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- unsigned shiftVal =
- ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &
- 0xff;
- unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
-
- if (shiftVal >= numElems)
- return builder.getNullValue(ops[0].getType(), loc);
-
- mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
-
- SmallVector<mlir::Attribute, 64> indices;
- mlir::Type i32Ty = builder.getSInt32Ty();
- for (auto i : llvm::seq<unsigned>(0, numElems))
- indices.push_back(cir::IntAttr::get(i32Ty, i + shiftVal));
-
- mlir::Value zero = builder.getNullValue(in.getType(), loc);
- mlir::Value sv = builder.createVecShuffle(loc, in, zero, indices);
- return builder.createBitcast(sv, ops[0].getType());
- }
- case X86::BI__builtin_ia32_vprotbi:
- case X86::BI__builtin_ia32_vprotwi:
- case X86::BI__builtin_ia32_vprotdi:
- case X86::BI__builtin_ia32_vprotqi:
- case X86::BI__builtin_ia32_prold128:
- case X86::BI__builtin_ia32_prold256:
- case X86::BI__builtin_ia32_prold512:
- case X86::BI__builtin_ia32_prolq128:
- case X86::BI__builtin_ia32_prolq256:
- case X86::BI__builtin_ia32_prolq512:
- return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
- ops[0], ops[1], false);
- case X86::BI__builtin_ia32_prord128:
- case X86::BI__builtin_ia32_prord256:
- case X86::BI__builtin_ia32_prord512:
- case X86::BI__builtin_ia32_prorq128:
- case X86::BI__builtin_ia32_prorq256:
- case X86::BI__builtin_ia32_prorq512:
- return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
- ops[0], ops[1], true);
- case X86::BI__builtin_ia32_selectb_128:
- case X86::BI__builtin_ia32_selectb_256:
- case X86::BI__builtin_ia32_selectb_512:
- case X86::BI__builtin_ia32_selectw_128:
- case X86::BI__builtin_ia32_selectw_256:
- case X86::BI__builtin_ia32_selectw_512:
- case X86::BI__builtin_ia32_selectd_128:
- case X86::BI__builtin_ia32_selectd_256:
- case X86::BI__builtin_ia32_selectd_512:
- case X86::BI__builtin_ia32_selectq_128:
- case X86::BI__builtin_ia32_selectq_256:
- case X86::BI__builtin_ia32_selectq_512:
- case X86::BI__builtin_ia32_selectph_128:
- case X86::BI__builtin_ia32_selectph_256:
- case X86::BI__builtin_ia32_selectph_512:
- case X86::BI__builtin_ia32_selectpbf_128:
- case X86::BI__builtin_ia32_selectpbf_256:
- case X86::BI__builtin_ia32_selectpbf_512:
- case X86::BI__builtin_ia32_selectps_128:
- case X86::BI__builtin_ia32_selectps_256:
- case X86::BI__builtin_ia32_selectps_512:
- case X86::BI__builtin_ia32_selectpd_128:
- case X86::BI__builtin_ia32_selectpd_256:
- case X86::BI__builtin_ia32_selectpd_512:
- return emitX86Select(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
- ops[2]);
- case X86::BI__builtin_ia32_selectsh_128:
- case X86::BI__builtin_ia32_selectsbf_128:
- case X86::BI__builtin_ia32_selectss_128:
- case X86::BI__builtin_ia32_selectsd_128:
- return emitX86ScalarSelect(builder, getLoc(expr->getExprLoc()), ops[0],
- ops[1], ops[2]);
- case X86::BI__builtin_ia32_cmpb128_mask:
- case X86::BI__builtin_ia32_cmpb256_mask:
- case X86::BI__builtin_ia32_cmpb512_mask:
- case X86::BI__builtin_ia32_cmpw128_mask:
- case X86::BI__builtin_ia32_cmpw256_mask:
- case X86::BI__builtin_ia32_cmpw512_mask:
- case X86::BI__builtin_ia32_cmpd128_mask:
- case X86::BI__builtin_ia32_cmpd256_mask:
- case X86::BI__builtin_ia32_cmpd512_mask:
- case X86::BI__builtin_ia32_cmpq128_mask:
- case X86::BI__builtin_ia32_cmpq256_mask:
- case X86::BI__builtin_ia32_cmpq512_mask:
- case X86::BI__builtin_ia32_ucmpb128_mask:
- case X86::BI__builtin_ia32_ucmpb256_mask:
- case X86::BI__builtin_ia32_ucmpb512_mask:
- case X86::BI__builtin_ia32_ucmpw128_mask:
- case X86::BI__builtin_ia32_ucmpw256_mask:
- case X86::BI__builtin_ia32_ucmpw512_mask:
- case X86::BI__builtin_ia32_ucmpd128_mask:
- case X86::BI__builtin_ia32_ucmpd256_mask:
- case X86::BI__builtin_ia32_ucmpd512_mask:
- case X86::BI__builtin_ia32_ucmpq128_mask:
- case X86::BI__builtin_ia32_ucmpq256_mask:
- case X86::BI__builtin_ia32_ucmpq512_mask:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_vpcomb:
- case X86::BI__builtin_ia32_vpcomw:
- case X86::BI__builtin_ia32_vpcomd:
- case X86::BI__builtin_ia32_vpcomq:
- return emitX86vpcom(builder, getLoc(expr->getExprLoc()), ops, true);
- case X86::BI__builtin_ia32_vpcomub:
- case X86::BI__builtin_ia32_vpcomuw:
- case X86::BI__builtin_ia32_vpcomud:
- case X86::BI__builtin_ia32_vpcomuq:
- return emitX86vpcom(builder, getLoc(expr->getExprLoc()), ops, false);
- case X86::BI__builtin_ia32_kortestcqi:
- case X86::BI__builtin_ia32_kortestchi:
- case X86::BI__builtin_ia32_kortestcsi:
- case X86::BI__builtin_ia32_kortestcdi: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- cir::IntType ty = cast<cir::IntType>(ops[0].getType());
- mlir::Value allOnesOp =
- builder.getConstAPInt(loc, ty, APInt::getAllOnes(ty.getWidth()));
- mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
- mlir::Value cmp =
- cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allOnesOp);
- return builder.createCast(cir::CastKind::bool_to_int, cmp,
- cgm.convertType(expr->getType()));
- }
- case X86::BI__builtin_ia32_kortestzqi:
- case X86::BI__builtin_ia32_kortestzhi:
- case X86::BI__builtin_ia32_kortestzsi:
- case X86::BI__builtin_ia32_kortestzdi: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- cir::IntType ty = cast<cir::IntType>(ops[0].getType());
- mlir::Value allZerosOp = builder.getNullValue(ty, loc).getResult();
- mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
- mlir::Value cmp =
- cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allZerosOp);
- return builder.createCast(cir::CastKind::bool_to_int, cmp,
- cgm.convertType(expr->getType()));
- }
- case X86::BI__builtin_ia32_ktestcqi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestc.b", ops);
- case X86::BI__builtin_ia32_ktestzqi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestz.b", ops);
- case X86::BI__builtin_ia32_ktestchi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestc.w", ops);
- case X86::BI__builtin_ia32_ktestzhi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestz.w", ops);
- case X86::BI__builtin_ia32_ktestcsi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestc.d", ops);
- case X86::BI__builtin_ia32_ktestzsi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestz.d", ops);
- case X86::BI__builtin_ia32_ktestcdi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestc.q", ops);
- case X86::BI__builtin_ia32_ktestzdi:
- return emitX86MaskTest(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.ktestz.q", ops);
- case X86::BI__builtin_ia32_kaddqi:
- return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kadd.b", ops);
- case X86::BI__builtin_ia32_kaddhi:
- return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kadd.w", ops);
- case X86::BI__builtin_ia32_kaddsi:
- return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kadd.d", ops);
- case X86::BI__builtin_ia32_kadddi:
- return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),
- "x86.avx512.kadd.q", ops);
- case X86::BI__builtin_ia32_kandqi:
- case X86::BI__builtin_ia32_kandhi:
- case X86::BI__builtin_ia32_kandsi:
- case X86::BI__builtin_ia32_kanddi:
- return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
- cir::BinOpKind::And, ops);
- case X86::BI__builtin_ia32_kandnqi:
- case X86::BI__builtin_ia32_kandnhi:
- case X86::BI__builtin_ia32_kandnsi:
- case X86::BI__builtin_ia32_kandndi:
- return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
- cir::BinOpKind::And, ops, true);
- case X86::BI__builtin_ia32_korqi:
- case X86::BI__builtin_ia32_korhi:
- case X86::BI__builtin_ia32_korsi:
- case X86::BI__builtin_ia32_kordi:
- return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
- cir::BinOpKind::Or, ops);
- case X86::BI__builtin_ia32_kxnorqi:
- case X86::BI__builtin_ia32_kxnorhi:
- case X86::BI__builtin_ia32_kxnorsi:
- case X86::BI__builtin_ia32_kxnordi:
- return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
- cir::BinOpKind::Xor, ops, true);
- case X86::BI__builtin_ia32_kxorqi:
- case X86::BI__builtin_ia32_kxorhi:
- case X86::BI__builtin_ia32_kxorsi:
- case X86::BI__builtin_ia32_kxordi:
- return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),
- cir::BinOpKind::Xor, ops);
- case X86::BI__builtin_ia32_knotqi:
- case X86::BI__builtin_ia32_knothi:
- case X86::BI__builtin_ia32_knotsi:
- case X86::BI__builtin_ia32_knotdi: {
- cir::IntType intTy = cast<cir::IntType>(ops[0].getType());
- unsigned numElts = intTy.getWidth();
- mlir::Value resVec =
- getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
- return builder.createBitcast(builder.createNot(resVec), ops[0].getType());
- }
- case X86::BI__builtin_ia32_kmovb:
- case X86::BI__builtin_ia32_kmovw:
- case X86::BI__builtin_ia32_kmovd:
- case X86::BI__builtin_ia32_kmovq: {
- // Bitcast to vXi1 type and then back to integer. This gets the mask
- // register type into the IR, but might be optimized out depending on
- // what's around it.
- cir::IntType intTy = cast<cir::IntType>(ops[0].getType());
- unsigned numElts = intTy.getWidth();
- mlir::Value resVec =
- getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
- return builder.createBitcast(resVec, ops[0].getType());
- }
- case X86::BI__builtin_ia32_sqrtsh_round_mask:
- case X86::BI__builtin_ia32_sqrtsd_round_mask:
- case X86::BI__builtin_ia32_sqrtss_round_mask:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_sqrtph512:
- case X86::BI__builtin_ia32_sqrtps512:
- case X86::BI__builtin_ia32_sqrtpd512: {
- mlir::Location loc = getLoc(expr->getExprLoc());
- mlir::Value arg = ops[0];
- return cir::SqrtOp::create(builder, loc, arg.getType(), arg).getResult();
- }
- case X86::BI__builtin_ia32_pmuludq128:
- case X86::BI__builtin_ia32_pmuludq256:
- case X86::BI__builtin_ia32_pmuludq512: {
- unsigned opTypePrimitiveSizeInBits =
- cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
- return emitX86Muldq(builder, getLoc(expr->getExprLoc()), /*isSigned*/ false,
- ops, opTypePrimitiveSizeInBits);
- }
- case X86::BI__builtin_ia32_pmuldq128:
- case X86::BI__builtin_ia32_pmuldq256:
- case X86::BI__builtin_ia32_pmuldq512: {
- unsigned opTypePrimitiveSizeInBits =
- cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
- return emitX86Muldq(builder, getLoc(expr->getExprLoc()), /*isSigned*/ true,
- ops, opTypePrimitiveSizeInBits);
- }
- case X86::BI__builtin_ia32_pternlogd512_mask:
- case X86::BI__builtin_ia32_pternlogq512_mask:
- case X86::BI__builtin_ia32_pternlogd128_mask:
- case X86::BI__builtin_ia32_pternlogd256_mask:
- case X86::BI__builtin_ia32_pternlogq128_mask:
- case X86::BI__builtin_ia32_pternlogq256_mask:
- case X86::BI__builtin_ia32_pternlogd512_maskz:
- case X86::BI__builtin_ia32_pternlogq512_maskz:
- case X86::BI__builtin_ia32_pternlogd128_maskz:
- case X86::BI__builtin_ia32_pternlogd256_maskz:
- case X86::BI__builtin_ia32_pternlogq128_maskz:
- case X86::BI__builtin_ia32_pternlogq256_maskz:
- case X86::BI__builtin_ia32_vpshldd128:
- case X86::BI__builtin_ia32_vpshldd256:
- case X86::BI__builtin_ia32_vpshldd512:
- case X86::BI__builtin_ia32_vpshldq128:
- case X86::BI__builtin_ia32_vpshldq256:
- case X86::BI__builtin_ia32_vpshldq512:
- case X86::BI__builtin_ia32_vpshldw128:
- case X86::BI__builtin_ia32_vpshldw256:
- case X86::BI__builtin_ia32_vpshldw512:
- case X86::BI__builtin_ia32_vpshrdd128:
- case X86::BI__builtin_ia32_vpshrdd256:
- case X86::BI__builtin_ia32_vpshrdd512:
- case X86::BI__builtin_ia32_vpshrdq128:
- case X86::BI__builtin_ia32_vpshrdq256:
- case X86::BI__builtin_ia32_vpshrdq512:
- case X86::BI__builtin_ia32_vpshrdw128:
- case X86::BI__builtin_ia32_vpshrdw256:
- case X86::BI__builtin_ia32_vpshrdw512:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return {};
- case X86::BI__builtin_ia32_reduce_fadd_pd512:
- case X86::BI__builtin_ia32_reduce_fadd_ps512:
- case X86::BI__builtin_ia32_reduce_fadd_ph512:
- case X86::BI__builtin_ia32_reduce_fadd_ph256:
- case X86::BI__builtin_ia32_reduce_fadd_ph128: {
- assert(!cir::MissingFeatures::fastMathFlags());
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "vector.reduce.fadd", ops[0].getType(),
- mlir::ValueRange{ops[0], ops[1]});
- }
- case X86::BI__builtin_ia32_reduce_fmul_pd512:
- case X86::BI__builtin_ia32_reduce_fmul_ps512:
- case X86::BI__builtin_ia32_reduce_fmul_ph512:
- case X86::BI__builtin_ia32_reduce_fmul_ph256:
- case X86::BI__builtin_ia32_reduce_fmul_ph128: {
- assert(!cir::MissingFeatures::fastMathFlags());
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "vector.reduce.fmul", ops[0].getType(),
- mlir::ValueRange{ops[0], ops[1]});
- }
- case X86::BI__builtin_ia32_reduce_fmax_pd512:
- case X86::BI__builtin_ia32_reduce_fmax_ps512:
- case X86::BI__builtin_ia32_reduce_fmax_ph512:
- case X86::BI__builtin_ia32_reduce_fmax_ph256:
- case X86::BI__builtin_ia32_reduce_fmax_ph128: {
- assert(!cir::MissingFeatures::fastMathFlags());
- cir::VectorType vecTy = cast<cir::VectorType>(ops[0].getType());
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "vector.reduce.fmax", vecTy.getElementType(),
- mlir::ValueRange{ops[0]});
- }
- case X86::BI__builtin_ia32_reduce_fmin_pd512:
- case X86::BI__builtin_ia32_reduce_fmin_ps512:
- case X86::BI__builtin_ia32_reduce_fmin_ph512:
- case X86::BI__builtin_ia32_reduce_fmin_ph256:
- case X86::BI__builtin_ia32_reduce_fmin_ph128: {
- assert(!cir::MissingFeatures::fastMathFlags());
- cir::VectorType vecTy = cast<cir::VectorType>(ops[0].getType());
- return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),
- "vector.reduce.fmin", vecTy.getElementType(),
- mlir::ValueRange{ops[0]});
- }
- case X86::BI__builtin_ia32_rdrand16_step:
- case X86::BI__builtin_ia32_rdrand32_step:
- case X86::BI__builtin_ia32_rdrand64_step:
- case X86::BI__builtin_ia32_rdseed16_step:
- case X86::BI__builtin_ia32_rdseed32_step:
- case X86::BI__builtin_ia32_rdseed64_step:
- case X86::BI__builtin_ia32_addcarryx_u32:
- case X86::BI__builtin_ia32_addcarryx_u64:
- case X86::BI__builtin_ia32_subborrow_u32:
- case X86::BI__builtin_ia32_subborrow_u64:
- case X86::BI__builtin_ia32_fpclassps128_mask:
- case X86::BI__builtin_ia32_fpclassps256_mask:
- case X86::BI__builtin_ia32_fpclassps512_mask:
- case X86::BI__builtin_ia32_vfpclassbf16128_mask:
- case X86::BI__builtin_ia32_vfpclassbf16256_mask:
- case X86::BI__builtin_ia32_vfpclassbf16512_mask:
- case X86::BI__builtin_ia32_fpclassph128_mask:
- case X86::BI__builtin_ia32_fpclassph256_mask:
- case X86::BI__builtin_ia32_fpclassph512_mask:
- case X86::BI__builtin_ia32_fpclasspd128_mask:
- case X86::BI__builtin_ia32_fpclasspd256_mask:
- case X86::BI__builtin_ia32_fpclasspd512_mask:
- case X86::BI__builtin_ia32_vp2intersect_q_512:
- case X86::BI__builtin_ia32_vp2intersect_q_256:
- case X86::BI__builtin_ia32_vp2intersect_q_128:
- case X86::BI__builtin_ia32_vp2intersect_d_512:
- case X86::BI__builtin_ia32_vp2intersect_d_256:
- case X86::BI__builtin_ia32_vp2intersect_d_128:
- case X86::BI__builtin_ia32_vpmultishiftqb128:
- case X86::BI__builtin_ia32_vpmultishiftqb256:
- case X86::BI__builtin_ia32_vpmultishiftqb512:
- case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
- case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
- case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
- case X86::BI__builtin_ia32_cmpeqps:
- case X86::BI__builtin_ia32_cmpeqpd:
- case X86::BI__builtin_ia32_cmpltps:
- case X86::BI__builtin_ia32_cmpltpd:
- case X86::BI__builtin_ia32_cmpleps:
- case X86::BI__builtin_ia32_cmplepd:
- case X86::BI__builtin_ia32_cmpunordps:
- case X86::BI__builtin_ia32_cmpunordpd:
- case X86::BI__builtin_ia32_cmpneqps:
- case X86::BI__builtin_ia32_cmpneqpd:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- case X86::BI__builtin_ia32_cmpnltps:
- case X86::BI__builtin_ia32_cmpnltpd:
- return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),
- cir::CmpOpKind::lt, /*shouldInvert=*/true);
- case X86::BI__builtin_ia32_cmpnleps:
- case X86::BI__builtin_ia32_cmpnlepd:
- return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),
- cir::CmpOpKind::le, /*shouldInvert=*/true);
- case X86::BI__builtin_ia32_cmpordps:
- case X86::BI__builtin_ia32_cmpordpd:
- case X86::BI__builtin_ia32_cmpph128_mask:
- case X86::BI__builtin_ia32_cmpph256_mask:
- case X86::BI__builtin_ia32_cmpph512_mask:
- case X86::BI__builtin_ia32_cmpps128_mask:
- case X86::BI__builtin_ia32_cmpps256_mask:
- case X86::BI__builtin_ia32_cmpps512_mask:
- case X86::BI__builtin_ia32_cmppd128_mask:
- case X86::BI__builtin_ia32_cmppd256_mask:
- case X86::BI__builtin_ia32_cmppd512_mask:
- case X86::BI__builtin_ia32_vcmpbf16512_mask:
- case X86::BI__builtin_ia32_vcmpbf16256_mask:
- case X86::BI__builtin_ia32_vcmpbf16128_mask:
- case X86::BI__builtin_ia32_cmpps:
- case X86::BI__builtin_ia32_cmpps256:
- case X86::BI__builtin_ia32_cmppd:
- case X86::BI__builtin_ia32_cmppd256:
- case X86::BI__builtin_ia32_cmpeqss:
- case X86::BI__builtin_ia32_cmpltss:
- case X86::BI__builtin_ia32_cmpless:
- case X86::BI__builtin_ia32_cmpunordss:
- case X86::BI__builtin_ia32_cmpneqss:
- case X86::BI__builtin_ia32_cmpnltss:
- case X86::BI__builtin_ia32_cmpnless:
- case X86::BI__builtin_ia32_cmpordss:
- case X86::BI__builtin_ia32_cmpeqsd:
- case X86::BI__builtin_ia32_cmpltsd:
- case X86::BI__builtin_ia32_cmplesd:
- case X86::BI__builtin_ia32_cmpunordsd:
- case X86::BI__builtin_ia32_cmpneqsd:
- case X86::BI__builtin_ia32_cmpnltsd:
- case X86::BI__builtin_ia32_cmpnlesd:
- case X86::BI__builtin_ia32_cmpordsd:
- case X86::BI__builtin_ia32_vcvtph2ps_mask:
- case X86::BI__builtin_ia32_vcvtph2ps256_mask:
- case X86::BI__builtin_ia32_vcvtph2ps512_mask:
- case X86::BI__builtin_ia32_cvtneps2bf16_128_mask:
- case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
- case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
- case X86::BI__cpuid:
- case X86::BI__cpuidex:
- case X86::BI__emul:
- case X86::BI__emulu:
- case X86::BI__mulh:
- case X86::BI__umulh:
- case X86::BI_mul128:
- case X86::BI_umul128: {
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- }
- case X86::BI__faststorefence: {
- cir::AtomicFenceOp::create(
- builder, getLoc(expr->getExprLoc()),
- cir::MemOrder::SequentiallyConsistent,
- cir::SyncScopeKindAttr::get(&getMLIRContext(),
- cir::SyncScopeKind::System));
- return mlir::Value{};
- }
- case X86::BI__shiftleft128:
- case X86::BI__shiftright128: {
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
- }
- case X86::BI_ReadWriteBarrier:
- case X86::BI_ReadBarrier:
- case X86::BI_WriteBarrier: {
- cir::AtomicFenceOp::create(
- builder, getLoc(expr->getExprLoc()),
- cir::MemOrder::SequentiallyConsistent,
- cir::SyncScopeKindAttr::get(&getMLIRContext(),
- cir::SyncScopeKind::SingleThread));
- return mlir::Value{};
- }
- case X86::BI_AddressOfReturnAddress:
- case X86::BI__stosb:
- case X86::BI__ud2:
- case X86::BI__int2c:
- case X86::BI__readfsbyte:
- case X86::BI__readfsword:
- case X86::BI__readfsdword:
- case X86::BI__readfsqword:
- case X86::BI__readgsbyte:
- case X86::BI__readgsword:
- case X86::BI__readgsdword:
- case X86::BI__readgsqword:
- case X86::BI__builtin_ia32_encodekey128_u32:
- case X86::BI__builtin_ia32_encodekey256_u32:
- case X86::BI__builtin_ia32_aesenc128kl_u8:
- case X86::BI__builtin_ia32_aesdec128kl_u8:
- case X86::BI__builtin_ia32_aesenc256kl_u8:
- case X86::BI__builtin_ia32_aesdec256kl_u8:
- case X86::BI__builtin_ia32_aesencwide128kl_u8:
- case X86::BI__builtin_ia32_aesdecwide128kl_u8:
- case X86::BI__builtin_ia32_aesencwide256kl_u8:
- case X86::BI__builtin_ia32_aesdecwide256kl_u8:
- case X86::BI__builtin_ia32_vfcmaddcph512_mask:
- case X86::BI__builtin_ia32_vfmaddcph512_mask:
- case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
- case X86::BI__builtin_ia32_vfmaddcsh_round_mask:
- case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
- case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
- case X86::BI__builtin_ia32_prefetchi:
- cgm.errorNYI(expr->getSourceRange(),
- std::string("unimplemented X86 builtin call: ") +
- getContext().BuiltinInfo.getName(builtinID));
- return mlir::Value{};
}
-}
More information about the cfe-commits
mailing list