[clang] [llvm] [mlir] [IRBuilder] Refactor for intrinsics const-folding (NFC) (PR #202738)
Ramkumar Ramachandra via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 9 23:47:14 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/202738
>From b806ade89d61a0e0cc0552b6035433231815c5b9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 9 Jun 2026 15:07:07 +0100
Subject: [PATCH 1/3] [IRBuilder] Refactor for intrinsics const-folding (NFC)
In preparation to const-fold intrinsic calls, refactor the IRBuilder
API, generalizing it to return possibly constant-folded values.
---
llvm/include/llvm/IR/IRBuilder.h | 182 +++++-----
llvm/lib/CodeGen/InlineAsmPrepare.cpp | 4 +-
llvm/lib/CodeGen/SafeStack.cpp | 5 +-
llvm/lib/IR/AutoUpgrade.cpp | 11 +-
llvm/lib/IR/IRBuilder.cpp | 340 ++++++++++--------
.../Target/AArch64/AArch64ISelLowering.cpp | 9 +-
.../Target/AArch64/AArch64StackTagging.cpp | 8 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 14 +-
llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp | 4 +-
.../AMDGPU/AMDGPUAsanInstrumentation.cpp | 3 +-
.../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 4 +-
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 3 +-
.../AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp | 3 +-
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 36 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 8 +-
.../Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 35 +-
.../AMDGPU/AMDGPULowerKernelArguments.cpp | 4 +-
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 14 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 10 +-
.../Target/ARM/MVEGatherScatterLowering.cpp | 92 ++---
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 15 +-
.../Target/Hexagon/HexagonVectorCombine.cpp | 13 +-
llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp | 4 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 8 +-
.../Target/RISCV/RISCVInterleavedAccess.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 147 ++++----
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 2 +-
.../Target/SPIRV/SPIRVLegalizePointerCast.cpp | 6 +-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 3 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 +-
llvm/lib/Target/SystemZ/SystemZTDC.cpp | 2 +-
.../Target/X86/X86InstCombineIntrinsic.cpp | 2 +-
llvm/lib/Target/X86/X86LowerAMXType.cpp | 13 +-
llvm/lib/Transforms/IPO/FatLTOCleanup.cpp | 2 +-
.../InstCombine/InstCombineAndOrXor.cpp | 2 +-
.../InstCombine/InstCombineCalls.cpp | 5 +-
.../InstCombine/InstCombineCompares.cpp | 4 +-
.../InstCombine/InstCombineMulDivRem.cpp | 10 +-
.../InstCombine/InstCombineSelect.cpp | 5 +-
.../InstCombine/InstructionCombining.cpp | 3 +-
.../Instrumentation/BoundsChecking.cpp | 6 +-
.../Instrumentation/MemorySanitizer.cpp | 33 +-
.../Transforms/Scalar/LoopIdiomRecognize.cpp | 32 +-
llvm/lib/Transforms/Utils/InlineFunction.cpp | 6 +-
.../Utils/ScalarEvolutionExpander.cpp | 6 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 12 +-
.../lib/Transforms/Utils/SimplifyLibCalls.cpp | 5 +-
.../Vectorize/LoadStoreVectorizer.cpp | 8 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 86 +++--
.../Transforms/Vectorize/VectorCombine.cpp | 14 +-
llvm/unittests/IR/IRBuilderTest.cpp | 38 +-
llvm/unittests/IR/IntrinsicsTest.cpp | 13 +-
llvm/unittests/Transforms/Utils/LocalTest.cpp | 4 +-
55 files changed, 704 insertions(+), 632 deletions(-)
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 99e4b748425c0..967a3fd8941d6 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -785,61 +785,61 @@ class IRBuilderBase {
uint32_t ElementSize, const AAMDNodes &AAInfo = AAMDNodes());
private:
- CallInst *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
+ Value *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
public:
/// Create a sequential vector fadd reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value. An unordered reduction
/// can be created by adding the reassoc fast-math flag to the resulting
/// sequential reduction.
- LLVM_ABI CallInst *CreateFAddReduce(Value *Acc, Value *Src);
+ LLVM_ABI Value *CreateFAddReduce(Value *Acc, Value *Src);
/// Create a sequential vector fmul reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value. An unordered reduction
/// can be created by adding the reassoc fast-math flag to the resulting
/// sequential reduction.
- LLVM_ABI CallInst *CreateFMulReduce(Value *Acc, Value *Src);
+ LLVM_ABI Value *CreateFMulReduce(Value *Acc, Value *Src);
/// Create a vector int add reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateAddReduce(Value *Src);
+ LLVM_ABI Value *CreateAddReduce(Value *Src);
/// Create a vector int mul reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateMulReduce(Value *Src);
+ LLVM_ABI Value *CreateMulReduce(Value *Src);
/// Create a vector int AND reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateAndReduce(Value *Src);
+ LLVM_ABI Value *CreateAndReduce(Value *Src);
/// Create a vector int OR reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateOrReduce(Value *Src);
+ LLVM_ABI Value *CreateOrReduce(Value *Src);
/// Create a vector int XOR reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateXorReduce(Value *Src);
+ LLVM_ABI Value *CreateXorReduce(Value *Src);
/// Create a vector integer max reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
+ LLVM_ABI Value *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
/// Create a vector integer min reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
+ LLVM_ABI Value *CreateIntMinReduce(Value *Src, bool IsSigned = false);
/// Create a vector float max reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateFPMaxReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMaxReduce(Value *Src);
/// Create a vector float min reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateFPMinReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMinReduce(Value *Src);
/// Create a vector float maximum reduction intrinsic of the source
/// vector. This variant follows the NaN and signed zero semantic of
/// llvm.maximum intrinsic.
- LLVM_ABI CallInst *CreateFPMaximumReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMaximumReduce(Value *Src);
/// Create a vector float minimum reduction intrinsic of the source
/// vector. This variant follows the NaN and signed zero semantic of
/// llvm.minimum intrinsic.
- LLVM_ABI CallInst *CreateFPMinimumReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMinimumReduce(Value *Src);
/// Create a lifetime.start intrinsic.
LLVM_ABI CallInst *CreateLifetimeStart(Value *Ptr);
@@ -857,36 +857,34 @@ class IRBuilderBase {
LLVM_ABI CallInst *CreateThreadLocalAddress(Value *Ptr);
/// Create a call to Masked Load intrinsic
- LLVM_ABI CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Store intrinsic
- LLVM_ABI CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
- Value *Mask);
+ LLVM_ABI Value *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+ Value *Mask);
/// Create a call to Masked Gather intrinsic
- LLVM_ABI CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Scatter intrinsic
- LLVM_ABI CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs,
- Align Alignment,
- Value *Mask = nullptr);
+ LLVM_ABI Value *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
+ Value *Mask = nullptr);
/// Create a call to Masked Expand Load intrinsic
- LLVM_ABI CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
- MaybeAlign Align,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Compress Store intrinsic
- LLVM_ABI CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align,
- Value *Mask = nullptr);
+ LLVM_ABI Value *CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align,
+ Value *Mask = nullptr);
/// Return an all true boolean vector (mask) with \p NumElts lanes.
Value *getAllOnesMask(ElementCount NumElts) {
@@ -896,15 +894,15 @@ class IRBuilderBase {
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided condition will be true.
- LLVM_ABI CallInst *CreateAssumption(Value *Cond);
+ LLVM_ABI Value *CreateAssumption(Value *Cond);
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided operand bundles hold.
- LLVM_ABI CallInst *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
+ LLVM_ABI Value *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
/// Create a llvm.experimental.noalias.scope.decl intrinsic call.
- LLVM_ABI Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
- Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
+ LLVM_ABI Value *CreateNoAliasScopeDeclaration(Value *Scope);
+ Value *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
return CreateNoAliasScopeDeclaration(
MetadataAsValue::get(Context, ScopeTag));
}
@@ -1021,26 +1019,26 @@ class IRBuilderBase {
/// Create a call to intrinsic \p ID with \p Args, mangled using
/// \p OverloadTypes. If \p FMFSource is provided, copy fast-math-flags from
/// that instruction to the intrinsic.
- LLVM_ABI CallInst *CreateIntrinsic(Intrinsic::ID ID,
- ArrayRef<Type *> OverloadTypes,
- ArrayRef<Value *> Args,
- FMFSource FMFSource = {},
- const Twine &Name = "",
- ArrayRef<OperandBundleDef> OpBundles = {});
+ LLVM_ABI Value *CreateIntrinsic(Intrinsic::ID ID,
+ ArrayRef<Type *> OverloadTypes,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource = {},
+ const Twine &Name = "",
+ ArrayRef<OperandBundleDef> OpBundles = {});
/// Create a call to intrinsic \p ID with \p RetTy and \p Args. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
/// the intrinsic.
- LLVM_ABI CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
- ArrayRef<Value *> Args,
- FMFSource FMFSource = {},
- const Twine &Name = "");
+ LLVM_ABI Value *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource = {},
+ const Twine &Name = "");
/// Create a call to non-overloaded intrinsic \p ID with \p Args. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
/// the intrinsic.
- CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Args,
- FMFSource FMFSource = {}, const Twine &Name = "") {
+ Value *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Args,
+ FMFSource FMFSource = {}, const Twine &Name = "") {
return CreateIntrinsic(ID, /*Types=*/{}, Args, FMFSource, Name);
}
@@ -1125,49 +1123,49 @@ class IRBuilderBase {
}
/// Create a call to the arithmetic_fence intrinsic.
- CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
- const Twine &Name = "") {
+ Value *CreateArithmeticFence(Value *Val, Type *DstType,
+ const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
- const Twine &Name = "") {
+ Value *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
+ const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::vector_extract,
{DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, uint64_t Idx,
- const Twine &Name = "") {
+ Value *CreateExtractVector(Type *DstType, Value *SrcVec, uint64_t Idx,
+ const Twine &Name = "") {
return CreateExtractVector(DstType, SrcVec, getInt64(Idx), Name);
}
/// Create a call to the vector.insert intrinsic.
- CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
- Value *Idx, const Twine &Name = "") {
+ Value *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
+ Value *Idx, const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::vector_insert,
{DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
nullptr, Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
- uint64_t Idx, const Twine &Name = "") {
+ Value *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
+ uint64_t Idx, const Twine &Name = "") {
return CreateInsertVector(DstType, SrcVec, SubVec, getInt64(Idx), Name);
}
/// Create a call to llvm.stacksave
- CallInst *CreateStackSave(const Twine &Name = "") {
+ Value *CreateStackSave(const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
return CreateIntrinsic(Intrinsic::stacksave, {DL.getAllocaPtrType(Context)},
{}, nullptr, Name);
}
/// Create a call to llvm.stackrestore
- CallInst *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
+ Value *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::stackrestore, {Ptr->getType()}, {Ptr},
nullptr, Name);
}
@@ -1183,9 +1181,9 @@ class IRBuilderBase {
private:
/// Create a call to a masked intrinsic with given Id.
- CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name = "");
+ Value *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name = "");
//===--------------------------------------------------------------------===//
// Instruction creation methods: Terminators
@@ -1832,19 +1830,19 @@ class IRBuilderBase {
/// the created intrinsic call according to \p Rounding and \p
/// Except and it sets \p FPMathTag as the 'fpmath' metadata, using
/// defaults if a value equals nullopt/null.
- LLVM_ABI CallInst *CreateConstrainedFPIntrinsic(
+ LLVM_ABI Value *CreateConstrainedFPIntrinsic(
Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
- LLVM_ABI CallInst *CreateConstrainedFPBinOp(
+ LLVM_ABI Value *CreateConstrainedFPBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
- LLVM_ABI CallInst *CreateConstrainedFPUnroundedBinOp(
+ LLVM_ABI Value *CreateConstrainedFPUnroundedBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -1916,8 +1914,8 @@ class IRBuilderBase {
CallInst *CreateStructuredAlloca(Type *BaseType, const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
PointerType *PtrTy = DL.getAllocaPtrType(Context);
- CallInst *Output =
- CreateIntrinsic(Intrinsic::structured_alloca, {PtrTy}, {}, {}, Name);
+ auto *Output = cast<CallInst>(
+ CreateIntrinsic(Intrinsic::structured_alloca, {PtrTy}, {}, {}, Name));
Output->addRetAttr(
Attribute::get(getContext(), Attribute::ElementType, BaseType));
return Output;
@@ -2003,17 +2001,18 @@ class IRBuilderBase {
new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
}
- CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
- ArrayRef<Value *> Indices,
- const Twine &Name = "") {
+ Value *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
+ ArrayRef<Value *> Indices,
+ const Twine &Name = "") {
SmallVector<Value *> Args;
Args.push_back(PtrBase);
llvm::append_range(Args, Indices);
- CallInst *Output = CreateIntrinsic(Intrinsic::structured_gep,
- {PtrBase->getType()}, Args, {}, Name);
- Output->addParamAttr(
- 0, Attribute::get(getContext(), Attribute::ElementType, BaseType));
+ Value *Output = CreateIntrinsic(Intrinsic::structured_gep,
+ {PtrBase->getType()}, Args, {}, Name);
+ if (auto *CI = dyn_cast<CallInst>(Output))
+ CI->addParamAttr(
+ 0, Attribute::get(getContext(), Attribute::ElementType, BaseType));
return Output;
}
@@ -2352,7 +2351,7 @@ class IRBuilderBase {
return CreateCast(CastOp, V, DestTy, Name, FPMathTag);
}
- LLVM_ABI CallInst *CreateConstrainedFPCast(
+ LLVM_ABI Value *CreateConstrainedFPCast(
Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
@@ -2537,7 +2536,7 @@ class IRBuilderBase {
FMFSource FMFSource, bool IsSignaling);
public:
- LLVM_ABI CallInst *CreateConstrainedFPCmp(
+ LLVM_ABI Value *CreateConstrainedFPCmp(
Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
const Twine &Name = "",
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -2809,9 +2808,8 @@ class IRBuilderBase {
/// Helper function that creates an assume intrinsic call that
/// represents an alignment assumption on the provided pointer \p PtrValue
/// with offset \p OffsetValue and alignment value \p AlignValue.
- CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
- Value *PtrValue, Value *AlignValue,
- Value *OffsetValue);
+ Value *CreateAlignmentAssumptionHelper(const DataLayout &DL, Value *PtrValue,
+ Value *AlignValue, Value *OffsetValue);
public:
/// Create an assume intrinsic call that represents an alignment
@@ -2820,10 +2818,9 @@ class IRBuilderBase {
/// An optional offset can be provided, and if it is provided, the offset
/// must be subtracted from the provided pointer to get the pointer with the
/// specified alignment.
- LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- uint64_t Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue, uint64_t Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents an alignment
/// assumption on the provided pointer.
@@ -2834,19 +2831,18 @@ class IRBuilderBase {
///
/// This overload handles the condition where the Alignment is dependent
/// on an existing value rather than a static value.
- LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- Value *Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue, Value *Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents a dereferencable
/// assumption on the provided pointer.
- LLVM_ABI CallInst *CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue);
+ LLVM_ABI Value *CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue);
/// Create an assume intrinsic call that represents a nonnull assumption on
/// the provided pointer.
- LLVM_ABI CallInst *CreateNonnullAssumption(Value *PtrValue);
+ LLVM_ABI Value *CreateNonnullAssumption(Value *PtrValue);
};
/// This provides a uniform API for creating instructions and inserting
diff --git a/llvm/lib/CodeGen/InlineAsmPrepare.cpp b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
index 0bbf55c3d31e4..8db216c26f858 100644
--- a/llvm/lib/CodeGen/InlineAsmPrepare.cpp
+++ b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
@@ -188,8 +188,8 @@ static bool insertIntrinsicCalls(CallBrInst *CBR, DominatorTree &DT) {
continue;
Builder.SetInsertPoint(&*IndDest->begin());
- CallInst *Intrinsic = Builder.CreateIntrinsic(
- CBR->getType(), Intrinsic::callbr_landingpad, {CBR});
+ CallInst *Intrinsic = cast<CallInst>(Builder.CreateIntrinsic(
+ CBR->getType(), Intrinsic::callbr_landingpad, {CBR}));
SSAUpdate.AddAvailableValue(IndDest, Intrinsic);
updateSSA(DT, CBR, Intrinsic, SSAUpdate);
Changed = true;
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index 152dcca209298..2a4c143de34ca 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -556,9 +556,10 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
if (FrameAlignment > StackAlignment) {
// Re-align the base pointer according to the max requested alignment.
IRB.SetInsertPoint(BasePointer->getNextNode());
- BasePointer = IRB.CreateIntrinsic(
+ BasePointer = cast<Instruction>(IRB.CreateIntrinsic(
StackPtrTy, Intrinsic::ptrmask,
- {BasePointer, ConstantInt::get(AddrTy, ~(FrameAlignment.value() - 1))});
+ {BasePointer,
+ ConstantInt::get(AddrTy, ~(FrameAlignment.value() - 1))}));
}
IRB.SetInsertPoint(BasePointer->getNextNode());
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 0770f0f0ff060..ebd5886b881c0 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5127,7 +5127,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
return;
};
- CallInst *NewCall = nullptr;
+ Value *NewCall = nullptr;
switch (NewFn->getIntrinsicID()) {
default: {
DefaultCase();
@@ -5625,7 +5625,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
{OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
- NewCall->setAttributes(NewAttrs);
+ if (auto *CI = dyn_cast<CallInst>(NewCall))
+ CI->setAttributes(NewAttrs);
auto *MemCI = cast<MemIntrinsic>(NewCall);
// All mem intrinsics support dest alignment.
const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
@@ -5693,8 +5694,10 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
llvm_unreachable("Unexpected intrinsic ID");
}
// Previous metadata is still valid.
- NewCall->copyMetadata(*CI);
- NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ if (auto *NewCI = dyn_cast<CallInst>(NewCall)) {
+ NewCI->copyMetadata(*CI);
+ NewCI->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ }
break;
}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 876d642b98fb0..8f7e25fa58982 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -254,7 +254,7 @@ CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
Type *Tys[] = {Ptr->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(Intrinsic::memset, Tys, Ops);
+ auto *CI = cast<MemSetInst>(CreateIntrinsic(Intrinsic::memset, Tys, Ops));
if (Align)
cast<MemSetInst>(CI)->setDestAlignment(*Align);
@@ -269,10 +269,11 @@ CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
Type *Tys[] = {Dst->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(Intrinsic::memset_inline, Tys, Ops);
+ auto *CI =
+ cast<MemSetInst>(CreateIntrinsic(Intrinsic::memset_inline, Tys, Ops));
if (DstAlign)
- cast<MemSetInst>(CI)->setDestAlignment(*DstAlign);
+ CI->setDestAlignment(*DstAlign);
CI->setAAMetadata(AAInfo);
return CI;
}
@@ -284,10 +285,9 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
Type *Tys[] = {Ptr->getType(), Size->getType()};
- CallInst *CI =
- CreateIntrinsic(Intrinsic::memset_element_unordered_atomic, Tys, Ops);
-
- cast<AnyMemSetInst>(CI)->setDestAlignment(Alignment);
+ auto *CI = cast<AnyMemSetInst>(
+ CreateIntrinsic(Intrinsic::memset_element_unordered_atomic, Tys, Ops));
+ CI->setDestAlignment(Alignment);
CI->setAAMetadata(AAInfo);
return CI;
}
@@ -303,15 +303,14 @@ CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(IntrID, Tys, Ops);
+ auto *MCI = cast<MemTransferInst>(CreateIntrinsic(IntrID, Tys, Ops));
- auto* MCI = cast<MemTransferInst>(CI);
if (DstAlign)
MCI->setDestAlignment(*DstAlign);
if (SrcAlign)
MCI->setSourceAlignment(*SrcAlign);
MCI->setAAMetadata(AAInfo);
- return CI;
+ return MCI;
}
CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
@@ -324,15 +323,14 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI =
- CreateIntrinsic(Intrinsic::memcpy_element_unordered_atomic, Tys, Ops);
+ auto *AMCI = cast<AnyMemCpyInst>(
+ CreateIntrinsic(Intrinsic::memcpy_element_unordered_atomic, Tys, Ops));
// Set the alignment of the pointer args.
- auto *AMCI = cast<AnyMemCpyInst>(CI);
AMCI->setDestAlignment(DstAlign);
AMCI->setSourceAlignment(SrcAlign);
AMCI->setAAMetadata(AAInfo);
- return CI;
+ return AMCI;
}
/// isConstantOne - Return true only if val is constant int 1
@@ -423,8 +421,8 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI =
- CreateIntrinsic(Intrinsic::memmove_element_unordered_atomic, Tys, Ops);
+ auto *CI = cast<CallInst>(
+ CreateIntrinsic(Intrinsic::memmove_element_unordered_atomic, Tys, Ops));
// Set the alignment of the pointer args.
CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
@@ -433,80 +431,82 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
return CI;
}
-CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
+Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
Value *Ops[] = {Src};
Type *Tys[] = { Src->getType() };
return CreateIntrinsic(ID, Tys, Ops);
}
-CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
+Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
Value *Ops[] = {Acc, Src};
return CreateIntrinsic(Intrinsic::vector_reduce_fadd, {Src->getType()}, Ops);
}
-CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
+Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
Value *Ops[] = {Acc, Src};
return CreateIntrinsic(Intrinsic::vector_reduce_fmul, {Src->getType()}, Ops);
}
-CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
+Value *IRBuilderBase::CreateAddReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
}
-CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
+Value *IRBuilderBase::CreateMulReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
}
-CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
+Value *IRBuilderBase::CreateAndReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
}
-CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
+Value *IRBuilderBase::CreateOrReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
}
-CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
+Value *IRBuilderBase::CreateXorReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
}
-CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
+Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
auto ID =
IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
return getReductionIntrinsic(ID, Src);
}
-CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
+Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
auto ID =
IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
return getReductionIntrinsic(ID, Src);
}
-CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
}
-CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
}
-CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);
}
-CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
}
CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.start only applies to pointers.");
- return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr});
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr}));
}
CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.end only applies to pointers.");
- return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr});
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr}));
}
CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
@@ -522,7 +522,8 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
Value *Ops[] = {Size, Ptr};
// Fill in the single overloaded type: memory object type.
Type *ObjectPtr[1] = {Ptr->getType()};
- return CreateIntrinsic(Intrinsic::invariant_start, ObjectPtr, Ops);
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::invariant_start, ObjectPtr, Ops));
}
static MaybeAlign getAlign(Value *Ptr) {
@@ -536,8 +537,8 @@ static MaybeAlign getAlign(Value *Ptr) {
CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
"threadlocal_address only applies to thread local variables.");
- CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
- {Ptr->getType()}, {Ptr});
+ CallInst *CI = cast<CallInst>(CreateIntrinsic(
+ llvm::Intrinsic::threadlocal_address, {Ptr->getType()}, {Ptr}));
if (MaybeAlign A = getAlign(Ptr)) {
CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));
CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));
@@ -545,20 +546,19 @@ CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
return CI;
}
-CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
+Value *IRBuilderBase::CreateAssumption(Value *Cond) {
assert(Cond->getType() == getInt1Ty() &&
"an assumption condition must be of type i1");
return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, {Cond});
}
-CallInst *
-IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
+Value *IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
Value *Args[] = {ConstantInt::getTrue(getContext())};
return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, Args,
/*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
}
-Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
+Value *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
return CreateIntrinsic(Intrinsic::experimental_noalias_scope_decl, {},
{Scope});
}
@@ -572,9 +572,9 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru,
+ const Twine &Name) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
@@ -582,10 +582,12 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
PassThru = PoisonValue::get(Ty);
Type *OverloadedTypes[] = { Ty, PtrTy };
Value *Ops[] = {Ptr, Mask, PassThru};
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name);
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked Store intrinsic.
@@ -594,27 +596,29 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
/// \p Alignment - alignment of the destination location
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
- Align Alignment, Value *Mask) {
+Value *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+ Value *Mask) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *OverloadedTypes[] = { DataTy, PtrTy };
Value *Ops[] = {Val, Ptr, Mask};
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked intrinsic, with given intrinsic Id,
/// an array of operands - Ops, and an array of overloaded types -
/// OverloadedTypes.
-CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
- ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
+ ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name) {
return CreateIntrinsic(Id, OverloadedTypes, Ops, {}, Name);
}
@@ -627,10 +631,9 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
- Align Alignment, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+ Value *Mask, Value *PassThru,
+ const Twine &Name) {
auto *VecTy = cast<VectorType>(Ty);
ElementCount NumElts = VecTy->getElementCount();
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
@@ -647,10 +650,12 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
- OverloadedTypes, Name);
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
+ OverloadedTypes, Name);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked Scatter intrinsic.
@@ -660,8 +665,8 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
/// \p Align - alignment for one element
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
- Align Alignment, Value *Mask) {
+Value *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
+ Align Alignment, Value *Mask) {
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
auto *DataTy = cast<VectorType>(Data->getType());
ElementCount NumElts = PtrsTy->getElementCount();
@@ -674,10 +679,12 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to Masked Expand Load intrinsic
@@ -689,10 +696,10 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
- MaybeAlign Align, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
+ MaybeAlign Align, Value *Mask,
+ Value *PassThru,
+ const Twine &Name) {
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
if (!PassThru)
@@ -700,11 +707,13 @@ CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {Ty, PtrTy};
Value *Ops[] = {Ptr, Mask, PassThru};
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
- OverloadedTypes, Name);
- if (Align)
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *Align));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
+ OverloadedTypes, Name);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (Align)
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), *Align));
+ return V;
}
/// Create a call to Masked Compress Store intrinsic
@@ -713,20 +722,21 @@ CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
/// \p Align - alignment of \p Ptr
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align,
- Value *Mask) {
+Value *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align, Value *Mask) {
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {DataTy, PtrTy};
Value *Ops[] = {Val, Ptr, Mask};
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
- OverloadedTypes);
- if (Align)
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), *Align));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
+ OverloadedTypes);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (Align)
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), *Align));
+ return V;
}
template <typename T0>
@@ -884,7 +894,7 @@ CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
Type *Types[] = {ResultType};
Value *Args[] = {Statepoint};
- return CreateIntrinsic(ID, Types, Args, {}, Name);
+ return cast<CallInst>(CreateIntrinsic(ID, Types, Args, {}, Name));
}
CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
@@ -893,22 +903,24 @@ CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
Type *Types[] = {ResultType};
Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
- return CreateIntrinsic(Intrinsic::experimental_gc_relocate, Types, Args, {},
- Name);
+ return cast<CallInst>(CreateIntrinsic(Intrinsic::experimental_gc_relocate,
+ Types, Args, {}, Name));
}
CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
const Twine &Name) {
Type *PtrTy = DerivedPtr->getType();
- return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_base,
- {PtrTy, PtrTy}, {DerivedPtr}, {}, Name);
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_base,
+ {PtrTy, PtrTy}, {DerivedPtr}, {}, Name));
}
CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
const Twine &Name) {
Type *PtrTy = DerivedPtr->getType();
- return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_offset, {PtrTy},
- {DerivedPtr}, {}, Name);
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_offset, {PtrTy},
+ {DerivedPtr}, {}, Name));
}
Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
@@ -933,27 +945,26 @@ Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
}
-CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
- ArrayRef<Type *> OverloadTypes,
- ArrayRef<Value *> Args,
- FMFSource FMFSource, const Twine &Name,
- ArrayRef<OperandBundleDef> OpBundles) {
+Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
+ ArrayRef<Type *> OverloadTypes,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource, const Twine &Name,
+ ArrayRef<OperandBundleDef> OpBundles) {
Module *M = BB->getModule();
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTypes);
return createCallHelper(Fn, Args, Name, FMFSource, OpBundles);
}
-CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
- ArrayRef<Value *> Args,
- FMFSource FMFSource,
- const Twine &Name) {
+Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource, const Twine &Name) {
Module *M = BB->getModule();
SmallVector<Type *> ArgTys = llvm::map_to_vector(Args, &Value::getType);
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, RetTy, ArgTys);
return createCallHelper(Fn, Args, Name, FMFSource);
}
-CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
+Value *IRBuilderBase::CreateConstrainedFPBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
std::optional<fp::ExceptionBehavior> Except) {
@@ -962,14 +973,16 @@ CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C = CreateIntrinsic(ID, {L->getType()},
- {L, R, RoundingV, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ Value *C = CreateIntrinsic(ID, {L->getType()}, {L, R, RoundingV, ExceptV},
+ nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
-CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
+Value *IRBuilderBase::CreateConstrainedFPIntrinsic(
Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
std::optional<RoundingMode> Rounding,
@@ -983,13 +996,15 @@ CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
ExtArgs.push_back(RoundingV);
ExtArgs.push_back(ExceptV);
- CallInst *C = CreateIntrinsic(ID, Types, ExtArgs, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ Value *C = CreateIntrinsic(ID, Types, ExtArgs, nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
-CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
+Value *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag,
std::optional<fp::ExceptionBehavior> Except) {
@@ -997,10 +1012,12 @@ CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C =
+ Value *C =
CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
@@ -1019,7 +1036,7 @@ Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
llvm_unreachable("Unexpected opcode!");
}
-CallInst *IRBuilderBase::CreateConstrainedFPCast(
+Value *IRBuilderBase::CreateConstrainedFPCast(
Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
std::optional<fp::ExceptionBehavior> Except) {
@@ -1027,7 +1044,7 @@ CallInst *IRBuilderBase::CreateConstrainedFPCast(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C;
+ Value *C;
if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {
Value *RoundingV = getConstrainedFPRounding(Rounding);
C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
@@ -1036,10 +1053,12 @@ CallInst *IRBuilderBase::CreateConstrainedFPCast(
C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
Name);
- setConstrainedFPCallAttr(C);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
- if (isa<FPMathOperator>(C))
- setFPAttrs(C, FPMathTag, UseFMF);
+ if (isa<FPMathOperator>(CI))
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
@@ -1060,15 +1079,16 @@ Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
Name);
}
-CallInst *IRBuilderBase::CreateConstrainedFPCmp(
+Value *IRBuilderBase::CreateConstrainedFPCmp(
Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
Value *PredicateV = getConstrainedFPPredicate(P);
Value *ExceptV = getConstrainedFPExcept(Except);
- CallInst *C = CreateIntrinsic(ID, {L->getType()},
- {L, R, PredicateV, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
+ Value *C = CreateIntrinsic(ID, {L->getType()}, {L, R, PredicateV, ExceptV},
+ nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C))
+ setConstrainedFPCallAttr(CI);
return C;
}
@@ -1300,15 +1320,16 @@ Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
Value *DimV = getInt32(Dimension);
- CallInst *Fn =
- CreateIntrinsic(Intrinsic::preserve_array_access_index,
- {ResultType, BaseType}, {Base, DimV, LastIndexV});
- Fn->addParamAttr(
- 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ Value *V = CreateIntrinsic(Intrinsic::preserve_array_access_index,
+ {ResultType, BaseType}, {Base, DimV, LastIndexV});
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ CI->addParamAttr(
+ 0, Attribute::get(CI->getContext(), Attribute::ElementType, ElTy));
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ }
- return Fn;
+ return V;
}
Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
@@ -1318,12 +1339,13 @@ Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
auto *BaseType = Base->getType();
Value *DIIndex = getInt32(FieldIndex);
- CallInst *Fn = CreateIntrinsic(Intrinsic::preserve_union_access_index,
- {BaseType, BaseType}, {Base, DIIndex});
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ Value *V = CreateIntrinsic(Intrinsic::preserve_union_access_index,
+ {BaseType, BaseType}, {Base, DIIndex});
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
- return Fn;
+ return V;
}
Value *IRBuilderBase::CreatePreserveStructAccessIndex(
@@ -1339,15 +1361,15 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex(
GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
Value *DIIndex = getInt32(FieldIndex);
- CallInst *Fn =
- CreateIntrinsic(Intrinsic::preserve_struct_access_index,
- {ResultType, BaseType}, {Base, GEPIndex, DIIndex});
- Fn->addParamAttr(
- 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
-
- return Fn;
+ Value *V = CreateIntrinsic(Intrinsic::preserve_struct_access_index,
+ {ResultType, BaseType}, {Base, GEPIndex, DIIndex});
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ CI->addParamAttr(
+ 0, Attribute::get(CI->getContext(), Attribute::ElementType, ElTy));
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ }
+ return V;
}
Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
@@ -1356,10 +1378,10 @@ Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
{FPNum, TestV});
}
-CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
- Value *PtrValue,
- Value *AlignValue,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
+ Value *PtrValue,
+ Value *AlignValue,
+ Value *OffsetValue) {
SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
if (OffsetValue)
Vals.push_back(OffsetValue);
@@ -1367,10 +1389,10 @@ CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
return CreateAssumption({AlignOpB});
}
-CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- uint64_t Alignment,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ uint64_t Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
assert(Alignment != 0 && "Invalid Alignment");
@@ -1380,17 +1402,17 @@ CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
}
-CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- Value *Alignment,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ Value *Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
}
-CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue) {
+Value *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a deferenceable assumption on a non-pointer?");
SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
@@ -1398,7 +1420,7 @@ CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
return CreateAssumption({DereferenceableOpB});
}
-CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
+Value *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a nonnull assumption on a non-pointer?");
return CreateAssumption(OperandBundleDef("nonnull", PtrValue));
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bb3d7b560b534..a81f265fa9d10 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -31352,10 +31352,11 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
const DataLayout &DL = M->getDataLayout();
IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
- CallInst *CI = Builder.CreateIntrinsic(Int, Tys, Addr);
- CI->addParamAttr(0, Attribute::get(Builder.getContext(),
- Attribute::ElementType, IntEltTy));
- Value *Trunc = Builder.CreateTrunc(CI, IntEltTy);
+ Value *Intr = Builder.CreateIntrinsic(Int, Tys, Addr);
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(0, Attribute::get(Builder.getContext(),
+ Attribute::ElementType, IntEltTy));
+ Value *Trunc = Builder.CreateTrunc(Intr, IntEltTy);
return Builder.CreateBitCast(Trunc, ValueTy);
}
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 7ef18fa18255d..447a8bfa67ff2 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -477,9 +477,9 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
assert(PrologueBB);
IRBuilder<> IRB(&PrologueBB->front());
- Instruction *Base =
+ Instruction *Base = cast<Instruction>(
IRB.CreateIntrinsic(Intrinsic::aarch64_irg_sp, {},
- {Constant::getNullValue(IRB.getInt64Ty())});
+ {Constant::getNullValue(IRB.getInt64Ty())}));
Base->setName("basetag");
const Triple &TargetTriple = M.getTargetTriple();
// This ABI will make it into Android API level 35.
@@ -583,10 +583,10 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
NextTag = (NextTag + 1) % 16;
// Replace alloca with tagp(alloca).
IRBuilder<> IRB(Info.AI->getNextNode());
- Instruction *TagPCall =
+ Instruction *TagPCall = cast<Instruction>(
IRB.CreateIntrinsic(Intrinsic::aarch64_tagp, {Info.AI->getType()},
{Constant::getNullValue(Info.AI->getType()), Base,
- ConstantInt::get(IRB.getInt64Ty(), Tag)});
+ ConstantInt::get(IRB.getInt64Ty(), Tag)}));
if (Info.AI->hasName())
TagPCall->setName(Info.AI->getName() + ".tag");
// Does not replace metadata, so we don't have to handle DbgVariableRecords.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4a55109bd2b23..14b151180f6a9 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2434,7 +2434,7 @@ instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
FMFSource = &II;
}
- CallInst *Res;
+ Value *Res;
if (MergeIntoAddendOp)
Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
{P, AddendOp, MulOp0, MulOp1}, FMFSource);
@@ -2457,10 +2457,11 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.replaceInstUsesWith(II, Load);
}
- CallInst *MaskedLoad =
+ Value *MaskedLoad =
IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
Pred, ConstantAggregateZero::get(VecTy));
- MaskedLoad->copyMetadata(II);
+ if (auto *NewI = dyn_cast<Instruction>(MaskedLoad))
+ NewI->copyMetadata(II);
return IC.replaceInstUsesWith(II, MaskedLoad);
}
@@ -2476,9 +2477,10 @@ instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.eraseInstFromFunction(II);
}
- CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
+ Value *MaskedStore = IC.Builder.CreateMaskedStore(
VecOp, PtrOp, PtrOp->getPointerAlignment(DL), Pred);
- MaskedStore->copyMetadata(II);
+ if (auto *NewI = dyn_cast<Instruction>(MaskedStore))
+ NewI->copyMetadata(II);
return IC.eraseInstFromFunction(II);
}
@@ -2720,7 +2722,7 @@ instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
BasePtr, IndexBase);
- CallInst *MaskedLoad =
+ Value *MaskedLoad =
IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
MaskedLoad->takeName(&II);
return IC.replaceInstUsesWith(II, MaskedLoad);
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index cf4f59bd6c304..600a2aa0cb946 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -160,9 +160,9 @@ bool SVEIntrinsicOpts::coalescePTrueIntrinsicCalls(
auto *MostEncompassingPTrueVTy =
cast<VectorType>(MostEncompassingPTrue->getType());
- auto *ConvertToSVBool = Builder.CreateIntrinsic(
+ auto *ConvertToSVBool = cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::aarch64_sve_convert_to_svbool, {MostEncompassingPTrueVTy},
- {MostEncompassingPTrue});
+ {MostEncompassingPTrue}));
bool ConvertFromCreated = false;
for (auto *PTrue : PTrues) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
index 9af3b05ff01fa..ad8dfee2a96a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
@@ -73,7 +73,8 @@ static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
IRB.SetInsertPoint(Trm);
- return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {});
+ return cast<Instruction>(
+ IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {}));
}
static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index cf4c2bbc78359..89d6df8fbacc2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -691,8 +691,8 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
// We need to know how many lanes are active within the wavefront, and we do
// this by doing a ballot of active lanes.
Type *const WaveTy = B.getIntNTy(ST.getWavefrontSize());
- CallInst *const Ballot =
- B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
+ CallInst *const Ballot = cast<CallInst>(
+ B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue()));
// We need to know how many lanes are active within the wavefront that are
// below us. If we counted each lane linearly starting from 0, a lane is
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index c5501236edb5b..a07752bd6817f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -2326,7 +2326,8 @@ INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
/// Create a workitem.id.x intrinsic call with range metadata.
CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
- CallInst *Tid = B.CreateIntrinsic(Intrinsic::amdgcn_workitem_id_x, {});
+ CallInst *Tid =
+ cast<CallInst>(B.CreateIntrinsic(Intrinsic::amdgcn_workitem_id_x, {}));
ST.makeLIDRangeMetadata(Tid);
return Tid;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index dd6858a15749e..3dbd81cb793e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -240,7 +240,8 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
Args[ImageDimIntr->DMaskIndex] =
ConstantInt::get(DMask->getType(), NewMaskVal);
Args[FragIdIndex] = ConstantInt::get(FragId->getType(), NewFragIdVal);
- CallInst *NewCall = B.CreateIntrinsic(NewIntrinID, OverloadTys, Args);
+ CallInst *NewCall =
+ cast<CallInst>(B.CreateIntrinsic(NewIntrinID, OverloadTys, Args));
LLVM_DEBUG(dbgs() << "Optimize: " << *NewCall << "\n");
NewCalls.push_back(NewCall);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index b17ccef685b21..90a7790b48aa0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -136,18 +136,20 @@ static std::optional<Instruction *> modifyIntrinsicCall(
// Modify arguments and types
Func(Args, OverloadTys);
- CallInst *NewCall = IC.Builder.CreateIntrinsic(NewIntr, OverloadTys, Args);
- NewCall->takeName(&OldIntr);
- NewCall->copyMetadata(OldIntr);
- if (isa<FPMathOperator>(NewCall))
- NewCall->copyFastMathFlags(&OldIntr);
- // Copy attributes
- AttributeList OldAttrList = OldIntr.getAttributes();
- NewCall->setAttributes(OldAttrList);
+ Value *V = IC.Builder.CreateIntrinsic(NewIntr, OverloadTys, Args);
+ V->takeName(&OldIntr);
+ if (auto *NewCall = dyn_cast<CallInst>(V)) {
+ NewCall->copyMetadata(OldIntr);
+ if (isa<FPMathOperator>(NewCall))
+ NewCall->copyFastMathFlags(&OldIntr);
+ // Copy attributes
+ AttributeList OldAttrList = OldIntr.getAttributes();
+ NewCall->setAttributes(OldAttrList);
+ }
// Erase and replace uses
if (!InstToReplace.getType()->isVoidTy())
- IC.replaceInstUsesWith(InstToReplace, NewCall);
+ IC.replaceInstUsesWith(InstToReplace, V);
bool RemoveOldIntr = &OldIntr != &InstToReplace;
@@ -1773,7 +1775,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Value *Args[] = {SrcLHS, SrcRHS,
ConstantInt::get(CC->getType(), SrcPred)};
- CallInst *NewCall = IC.Builder.CreateIntrinsic(
+ Value *NewCall = IC.Builder.CreateIntrinsic(
NewIID, {II.getType(), SrcLHS->getType()}, Args);
NewCall->takeName(&II);
return IC.replaceInstUsesWith(II, NewCall);
@@ -2211,7 +2213,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Args[0] = Src0;
Args[1] = Src1;
- CallInst *NewII = IC.Builder.CreateIntrinsic(
+ Value *NewII = IC.Builder.CreateIntrinsic(
IID, {Src0->getType(), Src1->getType()}, Args, &II);
NewII->takeName(&II);
return IC.replaceInstUsesWith(II, NewII);
@@ -2253,7 +2255,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Args[1] = Src0;
Args[3] = Src1;
- CallInst *NewII = IC.Builder.CreateIntrinsic(
+ Value *NewII = IC.Builder.CreateIntrinsic(
IID, {II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
Args, &II);
NewII->takeName(&II);
@@ -2403,12 +2405,14 @@ static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
Args[0] = IC.Builder.CreateShuffleVector(II.getOperand(0), EltMask);
}
- CallInst *NewCall =
+ Value *NewCall =
IC.Builder.CreateIntrinsic(II.getIntrinsicID(), OverloadTys, Args);
NewCall->takeName(&II);
- NewCall->copyMetadata(II);
- AttributeList OldAttrList = II.getAttributes();
- NewCall->setAttributes(OldAttrList);
+ if (auto *CI = dyn_cast<CallInst>(NewCall)) {
+ CI->copyMetadata(II);
+ AttributeList OldAttrList = II.getAttributes();
+ CI->setAttributes(OldAttrList);
+ }
if (IsLoad) {
if (NewNumElts == 1) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 679f7d5d8329f..b0ac4749a4044 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1804,7 +1804,8 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
auto *Call = IRB.CreateIntrinsic(IID, Ty, Args);
copyMetadata(Call, I);
- setAlign(Call, Alignment, Arg ? 1 : 0);
+ if (auto *CI = dyn_cast<CallInst>(Call))
+ setAlign(CI, Alignment, Arg ? 1 : 0);
Call->takeName(I);
insertPostMemOpFence(Order, SSID);
@@ -1874,7 +1875,8 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc,
Off, IRB.getInt32(0), IRB.getInt32(Aux)});
copyMetadata(Call, &AI);
- setAlign(Call, AI.getAlign(), 2);
+ if (auto *CI = dyn_cast<CallInst>(Call))
+ setAlign(CI, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
@@ -2326,7 +2328,7 @@ PtrParts SplitPtrStructs::visitIntrinsicInst(IntrinsicInst &I) {
IID == Intrinsic::amdgcn_load_to_lds
? Intrinsic::amdgcn_raw_ptr_buffer_load_lds
: Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds;
- Instruction *NewLoad = IRB.CreateIntrinsic(
+ Value *NewLoad = IRB.CreateIntrinsic(
NewIntr, {}, {Rsrc, LDSPtr, LoadSize, Off, SOffset, ImmOff, Aux});
copyMetadata(NewLoad, &I);
SplitUsers.insert(&I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index d9b018e3cefa0..b241517eb7df3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -105,17 +105,19 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// The default cluster barrier expects one signal per workgroup. So we need
// a workgroup barrier first.
if (IsSingleWaveWG) {
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {})
+ cast<Instruction>(
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {}))
->copyMetadata(I);
} else {
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
- CallInst *IsFirst = B.CreateIntrinsic(
+ auto *IsFirst = cast<Instruction>(B.CreateIntrinsic(
B.getInt1Ty(), Intrinsic::amdgcn_s_barrier_signal_isfirst,
- {BarrierID_32});
+ {BarrierID_32}));
IsFirst->copyMetadata(I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
+ cast<Instruction>(B.CreateIntrinsic(B.getVoidTy(),
+ Intrinsic::amdgcn_s_barrier_wait,
+ {BarrierID_16}))
->copyMetadata(I);
Instruction *ThenTerm =
@@ -127,13 +129,15 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// barrier in all waves.
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::CLUSTER);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::CLUSTER);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32})
+ cast<Instruction>(B.CreateIntrinsic(B.getVoidTy(),
+ Intrinsic::amdgcn_s_barrier_signal,
+ {BarrierID_32}))
->copyMetadata(I);
B.SetInsertPoint(&I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
+ cast<Instruction>(B.CreateIntrinsic(B.getVoidTy(),
+ Intrinsic::amdgcn_s_barrier_wait,
+ {BarrierID_16}))
->copyMetadata(I);
I.eraseFromParent();
@@ -160,7 +164,8 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// Down-grade waits, remove split signals.
if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {})
+ cast<Instruction>(
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {}))
->copyMetadata(I);
} else if (I.getIntrinsicID() ==
Intrinsic::amdgcn_s_barrier_signal_isfirst) {
@@ -176,11 +181,13 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// Lower to split barriers.
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32})
+ cast<Instruction>(B.CreateIntrinsic(B.getVoidTy(),
+ Intrinsic::amdgcn_s_barrier_signal,
+ {BarrierID_32}))
->copyMetadata(I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
+ cast<Instruction>(B.CreateIntrinsic(B.getVoidTy(),
+ Intrinsic::amdgcn_s_barrier_wait,
+ {BarrierID_16}))
->copyMetadata(I);
I.eraseFromParent();
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index a13011eca5a60..ff81741ebeb27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -221,9 +221,9 @@ static bool lowerKernelArguments(Function &F, const TargetMachine &TM,
if (TotalKernArgSize == 0)
return false;
- CallInst *KernArgSegment =
+ CallInst *KernArgSegment = cast<CallInst>(
Builder.CreateIntrinsic(Intrinsic::amdgcn_kernarg_segment_ptr, {},
- nullptr, F.getName() + ".kernarg.segment");
+ nullptr, F.getName() + ".kernarg.segment"));
KernArgSegment->addRetAttr(Attribute::NonNull);
KernArgSegment->addRetAttr(
Attribute::getWithDereferenceableBytes(Ctx, TotalKernArgSize));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 0981f6becffc4..60209af626c7d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -1203,10 +1203,10 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
if (!IsAMDHSA) {
- CallInst *LocalSizeY =
- Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_y, {});
- CallInst *LocalSizeZ =
- Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_z, {});
+ CallInst *LocalSizeY = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_y, {}));
+ CallInst *LocalSizeZ = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_z, {}));
ST.makeLIDRangeMetadata(LocalSizeY);
ST.makeLIDRangeMetadata(LocalSizeZ);
@@ -1248,8 +1248,8 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
// hsa_signal_t completion_signal; // uint64_t wrapper
// } hsa_kernel_dispatch_packet_t
//
- CallInst *DispatchPtr =
- Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {});
+ CallInst *DispatchPtr = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {}));
DispatchPtr->addRetAttr(Attribute::NoAlias);
DispatchPtr->addRetAttr(Attribute::NonNull);
F.removeFnAttr("amdgpu-no-dispatch-ptr");
@@ -1760,7 +1760,7 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(
case Intrinsic::objectsize: {
Value *Src = Intr->getOperand(0);
- CallInst *NewCall = Builder.CreateIntrinsic(
+ Value *NewCall = Builder.CreateIntrinsic(
Intrinsic::objectsize,
{Intr->getType(), PointerType::get(Context, AMDGPUAS::LOCAL_ADDRESS)},
{Src, Intr->getOperand(1), Intr->getOperand(2), Intr->getOperand(3)});
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6cc04c021d0b0..e1826f7823598 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20724,8 +20724,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
Value *LoadedShared = nullptr;
if (FullFlatEmulation) {
- CallInst *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
- {Addr}, nullptr, "is.shared");
+ Value *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
+ {Addr}, nullptr, "is.shared");
Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
Builder.SetInsertPoint(SharedBB);
Value *CastToLocal = Builder.CreateAddrSpaceCast(
@@ -20740,8 +20740,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
Builder.SetInsertPoint(CheckPrivateBB);
}
- CallInst *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
- {Addr}, nullptr, "is.private");
+ Value *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
+ {Addr}, nullptr, "is.private");
Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB);
Builder.SetInsertPoint(PrivateBB);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index e08a4f5ebda08..104fec5ea2cad 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -21403,7 +21403,8 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
Builder.getInt32(0), Builder.getInt32(7),
Builder.getInt32(10), Builder.getInt32(5)};
- return Builder.CreateIntrinsic(Intrinsic::arm_mcr, args);
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mcr, args));
} else {
// Instead of using barriers, atomic accesses on these subtargets use
// libcalls.
@@ -21413,7 +21414,8 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
// Only a full system barrier exists in the M-class architectures.
Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
Constant *CDomain = Builder.getInt32(Domain);
- return Builder.CreateIntrinsic(Intrinsic::arm_dmb, CDomain);
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_dmb, CDomain));
}
}
@@ -21692,7 +21694,7 @@ Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
Type *Tys[] = { Addr->getType() };
Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
- CallInst *CI = Builder.CreateIntrinsic(Int, Tys, Addr);
+ auto *CI = cast<CallInst>(Builder.CreateIntrinsic(Int, Tys, Addr));
CI->addParamAttr(
0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
@@ -21896,7 +21898,7 @@ bool ARMTargetLowering::lowerInterleavedLoad(
BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
VecTy->getNumElements() * Factor);
- CallInst *VldN = createLoadIntrinsic(BaseAddr);
+ Value *VldN = createLoadIntrinsic(BaseAddr);
// Replace uses of each shufflevector with the corresponding vector loaded
// by ldN.
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 869436a7e4964..5798f11bda16a 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -103,43 +103,41 @@ class MVEGatherScatterLowering : public FunctionPass {
// the other summand.
std::pair<Value *, int64_t> getVarAndConst(Value *Inst, int TypeScale);
- Instruction *lowerGather(IntrinsicInst *I);
+ Value *lowerGather(IntrinsicInst *I);
// Create a gather from a base + vector of offsets
- Instruction *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
- Instruction *&Root,
- IRBuilder<> &Builder);
+ Value *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
+ Instruction *&Root, IRBuilder<> &Builder);
// Create a gather from a vector of pointers
- Instruction *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder, int64_t Increment = 0);
// Create an incrementing gather from a vector of pointers
- Instruction *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
- Instruction *lowerScatter(IntrinsicInst *I);
+ Value *lowerScatter(IntrinsicInst *I);
// Create a scatter to a base + vector of offsets
- Instruction *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
- IRBuilder<> &Builder);
+ Value *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
+ IRBuilder<> &Builder);
// Create a scatter to a vector of pointers
- Instruction *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// Create an incrementing scatter from a vector of pointers
- Instruction *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// QI gathers and scatters can increment their offsets on their own if
// the increment is a constant value (digit)
- Instruction *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder);
+ Value *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder);
// QI gathers/scatters can increment their offsets on their own if the
// increment is a constant value (digit) - this creates a writeback QI
// gather/scatter
- Instruction *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
- Value *Ptr, unsigned TypeScale,
- IRBuilder<> &Builder);
+ Value *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
+ Value *Ptr, unsigned TypeScale,
+ IRBuilder<> &Builder);
// Optimise the base and offsets of the given address
bool optimiseAddress(Value *Address, BasicBlock *BB, LoopInfo *LI);
@@ -397,7 +395,7 @@ MVEGatherScatterLowering::getVarAndConst(Value *Inst, int TypeScale) {
return std::pair<Value *, int64_t>(Summand, Immediate);
}
-Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
+Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked gathers: checking transform preconditions\n"
<< *I << "\n");
@@ -423,7 +421,7 @@ Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
Instruction *Root = I;
- Instruction *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Value *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Load)
Load = tryCreateMaskedGatherOffset(I, Ptr, Root, Builder);
if (!Load)
@@ -450,8 +448,10 @@ Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
return Load;
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
- IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherBase(IntrinsicInst *I,
+ Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
LLVM_DEBUG(dbgs() << "masked gathers: loading from vector of pointers\n");
@@ -470,7 +470,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
{Ptr, Builder.getInt32(Increment), Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
@@ -491,7 +491,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
{Ptr, Builder.getInt32(Increment), Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
IntrinsicInst *I, Value *Ptr, Instruction *&Root, IRBuilder<> &Builder) {
using namespace PatternMatch;
@@ -553,7 +553,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
Root = Extend;
Value *Mask = I->getArgOperand(1);
- Instruction *Load = nullptr;
+ Value *Load = nullptr;
if (!match(Mask, m_One()))
Load = Builder.CreateIntrinsic(
Intrinsic::arm_mve_vldr_gather_offset_predicated,
@@ -574,7 +574,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
return Load;
}
-Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
+Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked scatters: checking transform preconditions\n"
<< *I << "\n");
@@ -598,7 +598,7 @@ Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
Builder.SetInsertPoint(I);
Builder.SetCurrentDebugLocation(I->getDebugLoc());
- Instruction *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Value *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Store)
Store = tryCreateMaskedScatterOffset(I, Ptr, Builder);
if (!Store)
@@ -612,7 +612,7 @@ Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
return Store;
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -636,7 +636,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
{Ptr, Builder.getInt32(Increment), Input, Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -658,7 +658,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
{Ptr, Builder.getInt32(Increment), Input, Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -723,7 +723,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
Builder.getInt32(Scale)});
}
-Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
+Value *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
FixedVectorType *Ty;
if (I->getIntrinsicID() == Intrinsic::masked_gather)
@@ -798,7 +798,7 @@ Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
return tryCreateMaskedScatterBase(I, OffsetsIncoming, Builder, Immediate);
}
-Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
+Value *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
IntrinsicInst *I, Value *BasePtr, Value *Offsets, unsigned TypeScale,
IRBuilder<> &Builder) {
// Check whether this gather's offset is incremented by a constant - if so,
@@ -860,8 +860,8 @@ Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
Builder.SetInsertPoint(I);
- Instruction *EndResult;
- Instruction *NewInduction;
+ Value *EndResult;
+ Value *NewInduction;
if (I->getIntrinsicID() == Intrinsic::masked_gather) {
// Build the incrementing gather
Value *Load = tryCreateMaskedGatherBaseWB(I, Phi, Builder, Immediate);
@@ -1281,22 +1281,22 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) {
}
}
for (IntrinsicInst *I : Gathers) {
- Instruction *L = lowerGather(I);
- if (L == nullptr)
+ Value *L = lowerGather(I);
+ if (!isa_and_nonnull<Instruction>(L))
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(L->getParent());
+ SimplifyInstructionsInBlock(cast<Instruction>(L)->getParent());
Changed = true;
}
for (IntrinsicInst *I : Scatters) {
- Instruction *S = lowerScatter(I);
- if (S == nullptr)
+ Value *S = lowerScatter(I);
+ if (!isa_and_nonnull<Instruction>(S))
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(S->getParent());
+ SimplifyInstructionsInBlock(cast<Instruction>(S)->getParent());
Changed = true;
}
return Changed;
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b12985d81fa75..4178bac9367cb 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -488,9 +488,8 @@ static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
return aspaceWrapValue(Cache, CI->getFunction(), P);
}
-static Instruction *aspaceMemSet(Intrinsic::ID ID,
- DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemSet(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -515,9 +514,8 @@ static Instruction *aspaceMemSet(Intrinsic::ID ID,
MI->getAAMetadata());
}
-static Instruction *aspaceMemCpy(Intrinsic::ID ID,
- DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemCpy(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -540,8 +538,7 @@ static Instruction *aspaceMemCpy(Intrinsic::ID ID,
IsVolatile, MI->getAAMetadata());
}
-static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemMove(DenseMap<Value *, Value *> &Cache, CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -620,7 +617,7 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
if (!IsSet && !IsCpy && !IsMove)
continue;
- Instruction *New;
+ Value *New;
if (IsSet)
New = aspaceMemSet(ID, CastsCache, CI);
else if (IsCpy)
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 80b6545861054..110d3abc8f417 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -803,10 +803,11 @@ auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
if (HVC.isTrue(Mask))
return createSimpleLoad(Builder, ValTy, Ptr, EffA.value(), MDSources);
- Instruction *Load =
+ Value *Load =
Builder.CreateMaskedLoad(ValTy, Ptr, EffA, Mask, PassThru, "mld");
LLVM_DEBUG(dbgs() << "\t[Creating masked Load:] "; Load->dump());
- propagateMetadata(Load, MDSources);
+ if (auto *LoadI = dyn_cast<Instruction>(Load))
+ propagateMetadata(LoadI, MDSources);
return Load;
}
@@ -871,9 +872,9 @@ auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
// Mask is unknown
if (!Predicate) {
- Instruction *Store =
- Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
- propagateMetadata(Store, MDSources);
+ Value *Store = Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
+ if (auto *StoreI = dyn_cast<Instruction>(Store))
+ propagateMetadata(StoreI, MDSources);
return Store;
}
@@ -2443,7 +2444,7 @@ Value *HvxIdioms::processVGather(Instruction &In) const {
return nullptr;
LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
- Instruction *Gather = nullptr;
+ Value *Gather = nullptr;
Type *NT = HVC.getHvxTy(HVC.getIntTy(32), false);
if (Qual == HvxIdioms::LdSt || Qual == HvxIdioms::Arithmetic) {
// We fully assume the address space is in VTCM. We also assume that all
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 42e65d7c9c847..1842a49a83e35 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -291,10 +291,10 @@ static void convertToParamAS(ArrayRef<Use *> OldUses, Value *Param) {
// alignment of the return value based on the alignment of the argument.
static CallInst *createNVVMInternalAddrspaceWrap(IRBuilder<> &IRB,
Argument &Arg) {
- CallInst *ArgInParam = IRB.CreateIntrinsic(
+ CallInst *ArgInParam = cast<CallInst>(IRB.CreateIntrinsic(
Intrinsic::nvvm_internal_addrspace_wrap,
{IRB.getPtrTy(ADDRESS_SPACE_ENTRY_PARAM), Arg.getType()}, &Arg, {},
- Arg.getName() + ".param");
+ Arg.getName() + ".param"));
if (MaybeAlign ParamAlign = Arg.getParamAlign())
ArgInParam->addRetAttr(
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index e43021b1f5379..3f91a65409b13 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -13062,8 +13062,8 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
// Other Lowering Code
//===----------------------------------------------------------------------===//
-static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
- return Builder.CreateIntrinsic(Id, {});
+static CallInst *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
+ return cast<CallInst>(Builder.CreateIntrinsic(Id, {}));
}
Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
@@ -13156,8 +13156,8 @@ Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
// http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
// and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
if (isa<LoadInst>(Inst))
- return Builder.CreateIntrinsic(Intrinsic::ppc_cfence, {Inst->getType()},
- {Inst});
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::ppc_cfence, {Inst->getType()}, {Inst}));
// FIXME: Can use isync for rmw operation.
return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
}
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 7026541cbd53d..55c10533c18db 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -239,7 +239,7 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
if (!isLegalInterleavedAccessType(VTy, MaskFactor, Alignment, AS, DL))
return false;
- CallInst *SegLoad = nullptr;
+ Value *SegLoad = nullptr;
if (MaskFactor < Factor && MaskFactor != 1) {
// Lower to strided segmented load.
unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index f5c462bdc629b..5549e067693be 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -179,9 +179,8 @@ static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
return std::nullopt;
}
-class SPIRVEmitIntrinsics
- : public ModulePass,
- public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
+class SPIRVEmitIntrinsics : public ModulePass,
+ public InstVisitor<SPIRVEmitIntrinsics, Value *> {
const SPIRVTargetMachine &TM;
SPIRVGlobalRegistry *GR = nullptr;
Function *CurrF = nullptr;
@@ -266,7 +265,7 @@ class SPIRVEmitIntrinsics
Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
bool IsPostprocessing);
- void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
+ void replaceMemInstrUses(Instruction *Old, Value *New, IRBuilder<> &B);
void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
bool UnknownElemTypeI8);
@@ -314,8 +313,8 @@ class SPIRVEmitIntrinsics
DenseMap<Function *, CallInst *> Ptrcasts);
void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
- void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
- Instruction *Dest, bool DeleteOld = true);
+ void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src, Value *Dest,
+ bool DeleteOld = true);
void applyDemangledPtrArgTypes(IRBuilder<> &B);
@@ -381,21 +380,21 @@ class SPIRVEmitIntrinsics
public:
static char ID;
SPIRVEmitIntrinsics(const SPIRVTargetMachine &TM) : ModulePass(ID), TM(TM) {}
- Instruction *visitInstruction(Instruction &I) { return &I; }
- Instruction *visitSwitchInst(SwitchInst &I);
- Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
- Instruction *visitIntrinsicInst(IntrinsicInst &I);
- Instruction *visitBitCastInst(BitCastInst &I);
- Instruction *visitInsertElementInst(InsertElementInst &I);
- Instruction *visitExtractElementInst(ExtractElementInst &I);
- Instruction *visitInsertValueInst(InsertValueInst &I);
- Instruction *visitExtractValueInst(ExtractValueInst &I);
- Instruction *visitLoadInst(LoadInst &I);
- Instruction *visitStoreInst(StoreInst &I);
- Instruction *visitAllocaInst(AllocaInst &I);
- Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
- Instruction *visitUnreachableInst(UnreachableInst &I);
- Instruction *visitCallInst(CallInst &I);
+ Value *visitInstruction(Instruction &I) { return &I; }
+ Value *visitSwitchInst(SwitchInst &I);
+ Value *visitGetElementPtrInst(GetElementPtrInst &I);
+ Value *visitIntrinsicInst(IntrinsicInst &I);
+ Value *visitBitCastInst(BitCastInst &I);
+ Value *visitInsertElementInst(InsertElementInst &I);
+ Value *visitExtractElementInst(ExtractElementInst &I);
+ Value *visitInsertValueInst(InsertValueInst &I);
+ Value *visitExtractValueInst(ExtractValueInst &I);
+ Value *visitLoadInst(LoadInst &I);
+ Value *visitStoreInst(StoreInst &I);
+ Value *visitAllocaInst(AllocaInst &I);
+ Value *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Value *visitUnreachableInst(UnreachableInst &I);
+ Value *visitCallInst(CallInst &I);
StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
@@ -528,15 +527,16 @@ void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
Instruction *Src,
- Instruction *Dest,
+ Value *Dest,
bool DeleteOld) {
replaceAllUsesWith(Src, Dest, DeleteOld);
std::string Name = Src->hasName() ? Src->getName().str() : "";
Src->eraseFromParent();
if (!Name.empty()) {
Dest->setName(Name);
- if (Named.insert(Dest).second)
- emitAssignName(Dest, B);
+ if (auto *DestI = dyn_cast<Instruction>(Dest))
+ if (Named.insert(Dest).second)
+ emitAssignName(DestI, B);
}
}
@@ -606,7 +606,7 @@ CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
SmallVector<Value *, 2> Args = {Op, buildMD(getNormalizedPoisonValue(ElemTy)),
B.getInt32(getPointerAddressSpace(OpTy))};
CallInst *PtrCasted =
- B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+ cast<CallInst>(B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args));
GR->buildAssignPtr(B, ElemTy, PtrCasted);
return PtrCasted;
}
@@ -893,7 +893,8 @@ SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
Args.push_back(B.getInt1(GEP.isInBounds()));
Args.push_back(GEP.getOperand(0));
llvm::append_range(Args, Indices);
- auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
+ auto *NewI =
+ cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}));
replaceAllUsesWithAndErase(B, &GEP, NewI);
return NewI;
}
@@ -1546,16 +1547,15 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(
TypeValidated.insert(I);
}
-void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
- Instruction *New,
+void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, Value *New,
IRBuilder<> &B) {
while (!Old->user_empty()) {
auto *U = Old->user_back();
if (isAssignTypeInstr(U)) {
B.SetInsertPoint(U);
SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
- CallInst *AssignCI =
- B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
+ CallInst *AssignCI = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::spv_assign_type, {New->getType()}, Args));
GR->addAssignPtrTypeInstr(New, AssignCI);
U->eraseFromParent();
} else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
@@ -1604,7 +1604,8 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
llvm_unreachable("illegal aggregate intrinsic user");
}
}
- New->copyMetadata(*Old);
+ if (auto *NewI = dyn_cast<Instruction>(New))
+ NewI->copyMetadata(*Old);
Old->eraseFromParent();
}
@@ -1632,7 +1633,8 @@ void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
setInsertPointSkippingPhis(B, I);
BPrepared = true;
}
- auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {});
+ auto *IntrUndef =
+ cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_undef, {}));
I->replaceUsesOfWith(Op, IntrUndef);
AggrConsts[IntrUndef] = AggrUndef;
AggrConstTypes[IntrUndef] = AggrUndef->getType();
@@ -1662,8 +1664,8 @@ void SPIRVEmitIntrinsics::preprocessPoisons(IRBuilder<> &B) {
setInsertPointSkippingPhis(B, &I);
BPrepared = true;
}
- auto *Call =
- B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = Poison;
AggrConstTypes[Call] = OpTy;
Replacement = Call;
@@ -1751,8 +1753,8 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
PrepareInsert();
Type *PoisonTy = Op->getType();
if (PoisonTy->isAggregateType()) {
- auto *Call = B.CreateIntrinsic(Intrinsic::spv_poison,
- {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(B.CreateIntrinsic(
+ Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = cast<PoisonValue>(Op);
AggrConstTypes[Call] = PoisonTy;
Op = Call;
@@ -1763,8 +1765,8 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
Args.push_back(Op);
}
PrepareInsert();
- auto *CI =
- B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args});
+ auto *CI = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args}));
Worklist.push(CI);
I->replaceUsesOfWith(Op, CI);
KeepInst = true;
@@ -1813,7 +1815,7 @@ static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
createSaturatedConversionDecoration(I, B);
}
-Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
+Value *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
if (!Call.isInlineAsm())
return &Call;
@@ -1872,7 +1874,7 @@ void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
createRoundingModeDecoration(FPI, RoundingModeDeco, B);
}
-Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
+Value *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
BasicBlock *ParentBB = I.getParent();
Function *F = ParentBB->getParent();
IRBuilder<> B(ParentBB);
@@ -1887,11 +1889,11 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
BBCases.push_back(Case.getCaseSuccessor());
Args.push_back(BlockAddress::get(F, Case.getCaseSuccessor()));
}
- CallInst *NewI = B.CreateIntrinsic(Intrinsic::spv_switch,
- {I.getOperand(0)->getType()}, {Args});
+ Value *NewV = B.CreateIntrinsic(Intrinsic::spv_switch,
+ {I.getOperand(0)->getType()}, {Args});
// remove switch to avoid its unneeded and undesirable unwrap into branches
// and conditions
- replaceAllUsesWith(&I, NewI);
+ replaceAllUsesWith(&I, NewV);
I.eraseFromParent();
// insert artificial and temporary instruction to preserve valid CFG,
// it will be removed after IR translation pass
@@ -1908,7 +1910,7 @@ static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
return GEP->getNumIndices() > 0 && match(GEP->getOperand(1), m_Zero());
}
-Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
+Value *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
auto *SGEP = dyn_cast<StructuredGEPInst>(&I);
if (!SGEP)
return &I;
@@ -1928,7 +1930,7 @@ Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
+Value *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
@@ -2035,7 +2037,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
+Value *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
Value *Source = I.getOperand(0);
@@ -2319,7 +2321,7 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
}
}
-Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
+Value *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
// If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
// type in LLT and IRTranslator will replace it by the scalar.
if (isVector1(I.getType()))
@@ -2336,8 +2338,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
return NewI;
}
-Instruction *
-SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
+Value *SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
// If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
// type in LLT and IRTranslator will replace it by the scalar.
if (isVector1(I.getVectorOperandType()))
@@ -2353,7 +2354,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
+Value *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
@@ -2366,13 +2367,12 @@ Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
Args.push_back(I.getInsertedValueOperand());
for (auto &Op : I.indices())
Args.push_back(B.getInt32(Op));
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
+Value *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
if (I.getAggregateOperand()->getType()->isAggregateType()) {
@@ -2392,7 +2392,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
+Value *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
if (!I.getType()->isAggregateType())
return &I;
IRBuilder<> B(I.getParent());
@@ -2411,14 +2411,14 @@ Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
IntrinsicId = Intrinsic::spv_atomic_load;
Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
}
- CallInst *NewI =
+ Value *NewI =
B.CreateIntrinsic(IntrinsicId, {I.getOperand(0)->getType()}, Args);
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
+Value *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
if (!AggrStores.contains(&I))
return &I;
IRBuilder<> B(I.getParent());
@@ -2451,14 +2451,15 @@ Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
IntrinsicId = Intrinsic::spv_atomic_store;
Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
}
- auto *NewI = B.CreateIntrinsic(
+ auto *NewV = B.CreateIntrinsic(
IntrinsicId, {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
- NewI->copyMetadata(I);
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
I.eraseFromParent();
- return NewI;
+ return NewV;
}
-Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
+Value *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
Value *ArraySize = nullptr;
if (I.isArrayAllocation()) {
const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
@@ -2485,7 +2486,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
+Value *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
assert(I.getType()->isAggregateType() && "Aggregate result is expected");
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
@@ -2544,7 +2545,7 @@ static bool precededByAbortIntrinsic(const UnreachableInst &I,
return false;
}
-Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
+Value *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
if (precededByAbortIntrinsic(I, ST))
return &I;
@@ -2583,7 +2584,7 @@ shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
IRBuilder<> &B) {
auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
- auto *Leaf = B.CreateIntrinsic(Intrinsic::spv_undef, {});
+ auto *Leaf = cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_undef, {}));
AggrConsts[Leaf] = PoisonValue::get(ElemTy);
AggrConstTypes[Leaf] = ElemTy;
return Leaf;
@@ -2602,8 +2603,8 @@ Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
Elems.push_back(Entry);
}
}
- auto *Composite = B.CreateIntrinsic(Intrinsic::spv_const_composite,
- {B.getInt32Ty()}, Elems);
+ auto *Composite = cast<Instruction>(B.CreateIntrinsic(
+ Intrinsic::spv_const_composite, {B.getInt32Ty()}, Elems));
AggrConsts[Composite] = PoisonValue::get(AggrTy);
AggrConstTypes[Composite] = AggrTy;
return Composite;
@@ -2629,8 +2630,8 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
isa<PoisonValue>(Init) &&
STI->canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze);
if (UsePoison) {
- auto *Call =
- B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = cast<PoisonValue>(Init);
AggrConstTypes[Call] = Init->getType();
InitOp = Call;
@@ -2640,8 +2641,8 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
}
Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType();
Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init;
- auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global,
- {GV.getType(), Ty}, {&GV, Const});
+ auto *InitInst = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::spv_init_global, {GV.getType(), Ty}, {&GV, Const}));
InitInst->setArgOperand(1, InitOp);
}
if (!Init && GV.use_empty())
@@ -3099,7 +3100,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
SmallVector<Value *, 2> Args = {
NewOp, buildMD(getNormalizedPoisonValue(OpElemTy)),
B.getInt32(getPointerAddressSpace(OpTy))};
- CallInst *PtrCasted =
+ Value *PtrCasted =
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
GR->buildAssignPtr(B, OpElemTy, PtrCasted);
NewOp = PtrCasted;
@@ -3585,8 +3586,10 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
setInsertPointAfterDef(B, I);
// Visitors return either the original/newly created instruction for
// further processing, nullptr otherwise.
- I = visit(*I);
- if (!I)
+ Value *Replacement = visit(*I);
+ if (isa_and_nonnull<Instruction>(Replacement))
+ I = cast<Instruction>(Replacement);
+ else
continue;
// Don't emit intrinsics for convergence operations.
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 343311fb44475..8bff367fc3c38 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -2229,7 +2229,7 @@ void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty,
MDString::get(Ctx, Arg->getName())};
B.CreateIntrinsic(Intrinsic::spv_value_md,
{MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});
- AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {Arg});
+ AssignCI = cast<CallInst>(B.CreateIntrinsic(Intrinsic::fake_use, {Arg}));
} else {
AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
OfType, Arg, {}, B);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
index cb03dcae21ede..5832d88af750c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
@@ -397,8 +397,7 @@ class SPIRVLegalizePointerCastImpl {
SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),
Element->getType(), Int32Ty};
SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)};
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
buildAssignType(B, Vector->getType(), NewI);
return NewI;
}
@@ -410,8 +409,7 @@ class SPIRVLegalizePointerCastImpl {
Type *Int32Ty = Type::getInt32Ty(B.getContext());
SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};
SmallVector<Value *> Args = {Vector, B.getInt32(Index)};
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
buildAssignType(B, ElementType, NewI);
return NewI;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 29fb4d1d9ec04..fec4374d9604b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -164,7 +164,8 @@ static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
F->getArg(0));
IRB.CreateRet(BSwap);
IntrinsicLowering IL(M->getDataLayout());
- IL.LowerIntrinsicCall(BSwap);
+ if (auto *CI = dyn_cast<CallInst>(BSwap))
+ IL.LowerIntrinsicCall(CI);
break;
}
default:
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 2b48986826b6a..02f5057e58828 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -997,7 +997,7 @@ CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
Args.push_back(Arg2);
Args.push_back(buildMD(Arg));
llvm::append_range(Args, Imms);
- return B.CreateIntrinsic(IntrID, {Types}, Args);
+ return cast<CallInst>(B.CreateIntrinsic(IntrID, {Types}, Args));
}
// Return true if there is an opaque pointer type nested in the argument.
diff --git a/llvm/lib/Target/SystemZ/SystemZTDC.cpp b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
index 1894e71792a3f..6948a8bdd9b6d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTDC.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
@@ -364,7 +364,7 @@ bool SystemZTDCPass::runOnFunction(Function &F) {
// Call the intrinsic, compare result with 0.
IRBuilder<> IRB(I);
Value *MaskVal = ConstantInt::get(Type::getInt64Ty(Ctx), Mask);
- Instruction *TDC =
+ Value *TDC =
IRB.CreateIntrinsic(Intrinsic::s390_tdc, V->getType(), {V, MaskVal});
Value *ICmp = IRB.CreateICmp(CmpInst::ICMP_NE, TDC, Zero32);
I->replaceAllUsesWith(ICmp);
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 4999581489e82..7eab4deb9a14c 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -68,7 +68,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
// intrinsic to the LLVM intrinsic to allow target-independent optimizations.
if (Value *BoolMask = getBoolVecFromMask(Mask, IC.getDataLayout())) {
// The pass-through vector for an x86 masked load is a zero vector.
- CallInst *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
+ Value *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
II.getType(), Ptr, Align(1), BoolMask, ZeroVec);
return IC.replaceInstUsesWith(II, NewMaskedLoad);
}
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index df42d1f4388ba..4be5b80481500 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -505,8 +505,8 @@ static Instruction *createTileStore(Instruction *TileDef, Value *Ptr) {
Value *Stride = Builder.getInt64(64);
std::array<Value *, 5> Args = {Row, Col, Ptr, Stride, TileDef};
- Instruction *TileStore =
- Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, Args);
+ Instruction *TileStore = cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, Args));
return TileStore;
}
@@ -824,11 +824,12 @@ bool X86LowerAMXCast::optimizeAMXCastFromPhi(
// Create tilezero at the end of incoming block.
auto *Block = OldPN->getIncomingBlock(I);
BasicBlock::iterator Iter = Block->getTerminator()->getIterator();
- Instruction *NewInst = Builder.CreateIntrinsic(
- Intrinsic::x86_tilezero_internal, {}, {Row, Col});
+ Instruction *NewInst = cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::x86_tilezero_internal, {}, {Row, Col}));
NewInst->moveBefore(Iter);
- NewInst = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
- {IncValue->getType()}, {NewInst});
+ NewInst = cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
+ {IncValue->getType()}, {NewInst}));
NewInst->moveBefore(Iter);
// Replace InValue with new Value.
OldPN->setIncomingValue(I, NewInst);
diff --git a/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp b/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
index f667187b76ed5..6e90896304b56 100644
--- a/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
+++ b/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
@@ -79,7 +79,7 @@ static bool cleanUpTypeCheckedLoad(Module &M, Function &CheckedLoadFn,
Value *Offset = I->getOperand(1);
Type *PtrTy = I->getType()->getStructElementType(0);
ConstantInt *True = ConstantInt::getTrue(M.getContext());
- Instruction *Load;
+ Value *Load;
if (IsRelative) {
Load =
IRB.CreateIntrinsic(Intrinsic::load_relative, {Offset->getType()},
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b4686a6a63e86..0f9b300912660 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1634,7 +1634,7 @@ Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
return replaceInstUsesWith(BO, II);
}
- CallInst *NewClass =
+ Value *NewClass =
Builder.CreateIntrinsic(Intrinsic::is_fpclass, {ClassVal0->getType()},
{ClassVal0, Builder.getInt32(NewClassMask)});
return replaceInstUsesWith(BO, NewClass);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 8efd1607780e9..c1ecd3203c261 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1600,7 +1600,7 @@ Value *InstCombinerImpl::foldReversedIntrinsicOperands(IntrinsicInst *II) {
// intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
- Instruction *NewIntrinsic = Builder.CreateIntrinsic(
+ Value *NewIntrinsic = Builder.CreateIntrinsic(
II->getType(), II->getIntrinsicID(), NewArgs, FPI);
return Builder.CreateVectorReverse(NewIntrinsic);
}
@@ -3035,8 +3035,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (ElementCount::isKnownGT(NegatedCount, RetCount)) {
SmallVector<Value *, 5> NewArgs(II->args());
NewArgs[NegatedOpArg] = OpNotNeg;
- Instruction *NewMul =
- Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
+ Value *NewMul = Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));
}
break;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 7b6d380acffe1..aa0fc6c652126 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -4945,7 +4945,7 @@ Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
if (MulHadOtherUses)
Builder.SetInsertPoint(Mul);
- CallInst *Call = Builder.CreateIntrinsic(
+ Value *Call = Builder.CreateIntrinsic(
Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
: Intrinsic::smul_with_overflow,
X->getType(), {X, Y}, /*FMFSource=*/nullptr, "mul");
@@ -6739,7 +6739,7 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
MulA = Builder.CreateZExt(A, MulType);
if (WidthB < MulWidth)
MulB = Builder.CreateZExt(B, MulType);
- CallInst *Call =
+ Value *Call =
Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, MulType,
{MulA, MulB}, /*FMFSource=*/nullptr, "umul");
IC.addToWorklist(MulInstr);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 17ac09c10f41c..a721d8eca7ac5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -636,7 +636,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
Value *Y, Value *Z) {
InstCombiner::BuilderTy &Builder = IC.Builder;
Value *YZ = Builder.CreateNSWAdd(Y, Z);
- Instruction *NewPow = Builder.CreateIntrinsic(
+ Value *NewPow = Builder.CreateIntrinsic(
Intrinsic::powi, {X->getType(), YZ->getType()}, {X, YZ}, &I);
return NewPow;
@@ -654,7 +654,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
m_Deferred(X)))) {
Constant *One = ConstantInt::get(Y->getType(), 1);
if (willNotOverflowSignedAdd(Y, One, I)) {
- Instruction *NewPow = createPowiExpr(I, *this, X, Y, One);
+ Value *NewPow = createPowiExpr(I, *this, X, Y, One);
return replaceInstUsesWith(I, NewPow);
}
}
@@ -668,7 +668,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
match(Op1, m_AllowReassoc(m_Intrinsic<Intrinsic::powi>(m_Specific(X),
m_Value(Z)))) &&
Y->getType() == Z->getType() && willNotOverflowSignedAdd(Y, Z, I)) {
- Instruction *NewPow = createPowiExpr(I, *this, X, Y, Z);
+ Value *NewPow = createPowiExpr(I, *this, X, Y, Z);
return replaceInstUsesWith(I, NewPow);
}
@@ -681,7 +681,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
m_Specific(Op1), m_Value(Y))))) &&
willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
- Instruction *NewPow = createPowiExpr(I, *this, Op1, Y, NegOne);
+ Value *NewPow = createPowiExpr(I, *this, Op1, Y, NegOne);
return replaceInstUsesWith(I, NewPow);
}
@@ -1927,7 +1927,7 @@ Instruction *InstCombinerImpl::foldFDivConstantDivisor(BinaryOperator &I) {
(match(I.getOperand(1), m_PosZeroFP()) ||
(I.hasNoSignedZeros() && match(I.getOperand(1), m_AnyZeroFP())))) {
IRBuilder<> B(&I);
- CallInst *CopySign = B.CreateIntrinsic(
+ Value *CopySign = B.CreateIntrinsic(
Intrinsic::copysign, {C->getType()},
{ConstantFP::getInfinity(I.getType()), I.getOperand(0)}, &I);
CopySign->takeName(&I);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index df9ba225657d3..ab2be55abd3b3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -417,9 +417,10 @@ Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
Value *SelectVal = Builder.CreateSelect(Cond, LdexpVal0, LdexpVal1);
Value *SelectExp = Builder.CreateSelect(Cond, LdexpExp0, LdexpExp1);
- CallInst *NewLdexp = Builder.CreateIntrinsic(
+ Value *NewLdexp = Builder.CreateIntrinsic(
TII->getType(), Intrinsic::ldexp, {SelectVal, SelectExp});
- NewLdexp->setFastMathFlags(FMF);
+ if (auto *NewI = dyn_cast<CallInst>(NewLdexp))
+ NewI->setFastMathFlags(FMF);
return replaceInstUsesWith(SI, NewLdexp);
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 608966a5e9289..5b6b156904612 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3978,7 +3978,8 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
MTI->getRawDest(),
ConstantInt::get(Type::getInt8Ty(MI.getContext()), 0),
MTI->getLength(), MTI->getDestAlign());
- M->copyMetadata(*MTI);
+ if (auto *MI = dyn_cast<Instruction>(M))
+ MI->copyMetadata(*MTI);
}
}
}
diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
index fb7e58f4632ef..03b4730b1be0b 100644
--- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -111,7 +111,7 @@ static Value *getBoundsCheckCond(Value *Ptr, Value *InstVal,
static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB,
std::optional<int8_t> GuardKind) {
if (!DebugTrapBB)
- return IRB.CreateIntrinsic(Intrinsic::trap, {});
+ return cast<CallInst>(IRB.CreateIntrinsic(Intrinsic::trap, {}));
uint64_t ImmArg = GuardKind.has_value()
? GuardKind.value()
@@ -121,8 +121,8 @@ static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB,
if (ImmArg > 255)
ImmArg = 255;
- return IRB.CreateIntrinsic(Intrinsic::ubsantrap,
- ConstantInt::get(IRB.getInt8Ty(), ImmArg));
+ return cast<CallInst>(IRB.CreateIntrinsic(
+ Intrinsic::ubsantrap, ConstantInt::get(IRB.getInt8Ty(), ImmArg)));
}
static CallInst *InsertCall(BuilderTy &IRB, bool MayReturn, StringRef Name) {
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index c45ec68f3cd07..b3da8c516c72b 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1266,9 +1266,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
removeUnreachableBlocks(F);
MS.initializeCallbacks(*F.getParent(), TLI);
- FnPrologueEnd =
+ FnPrologueEnd = cast<Instruction>(
IRBuilder<>(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt())
- .CreateIntrinsic(Intrinsic::donothing, {});
+ .CreateIntrinsic(Intrinsic::donothing, {}));
if (MS.CompileKernel) {
IRBuilder<> IRB(FnPrologueEnd);
@@ -4642,7 +4642,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// copy them.
ShadowArgs.append(1, IRB.CreateBitCast(SrcShadow, Src->getType()));
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -4697,7 +4697,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ShadowArgs.append(1, SrcShadowPtr);
ShadowArgs.append(1, Mask);
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(), ShadowArgs);
// The AVX masked load intrinsics do not have integer variants. We use the
// floating-point variants, which will happily copy the shadows even if
@@ -4747,8 +4747,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Shadows are integer-ish types but some intrinsics require a
// different (e.g., floating-point) type.
Shadow = IRB.CreateBitCast(Shadow, I.getArgOperand(0)->getType());
- CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {Shadow, I.getArgOperand(1)});
+ Value *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {Shadow, I.getArgOperand(1)});
setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
setOriginForNaryOp(I);
@@ -4781,8 +4781,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// different (e.g., floating-point) type.
AShadow = IRB.CreateBitCast(AShadow, I.getArgOperand(0)->getType());
BShadow = IRB.CreateBitCast(BShadow, I.getArgOperand(2)->getType());
- CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {AShadow, Idx, BShadow});
+ Value *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {AShadow, Idx, BShadow});
setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
setOriginForNaryOp(I);
}
@@ -5394,12 +5394,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Value *XShadow = getShadow(X);
Value *BZeroShadow = getCleanShadow(B);
- CallInst *AShadowXShadow = IRB.CreateIntrinsic(
+ Value *AShadowXShadow = IRB.CreateIntrinsic(
I.getType(), I.getIntrinsicID(), {XShadow, AShadow, BZeroShadow});
- CallInst *AShadowX = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {X, AShadow, BZeroShadow});
- CallInst *XShadowA = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {XShadow, A, BZeroShadow});
+ Value *AShadowX = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {X, AShadow, BZeroShadow});
+ Value *XShadowA = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {XShadow, A, BZeroShadow});
unsigned NumElements = cast<FixedVectorType>(I.getType())->getNumElements();
Value *BShadow = getShadow(B);
@@ -5477,7 +5477,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// The NEON vector load instructions handled by this function all have
// integer variants. It is easier to use those rather than trying to cast
// a struct of vectors of floats into a struct of vectors of integers.
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(getShadowTy(&I), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -5560,7 +5560,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Addr, IRB, OutputShadowTy, Align(1), /*isStore*/ true);
ShadowArgs.append(1, OutputShadowPtr);
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -5757,8 +5757,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ShadowArgs.push_back(Arg);
}
- CallInst *CI =
- IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
+ Value *CI = IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
Value *CombinedShadow = CI;
// Combine the computed shadow with the shadow of trailing args
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7a08d99a9e505..c8f56aaa04c0b 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1174,11 +1174,11 @@ bool LoopIdiomRecognize::processLoopStridedStore(
isLibFuncEmittable(M, TLI, LibFunc_memset_pattern16)) {
assert(isa<SCEVConstant>(StoreSizeSCEV) && "Expected constant store size");
- NewCall = Builder.CreateIntrinsic(
+ NewCall = cast<CallInst>(Builder.CreateIntrinsic(
Intrinsic::experimental_memset_pattern,
{DestInt8PtrTy, PatternValue->getType(), IntIdxTy},
{BasePtr, PatternValue, MemsetArg,
- ConstantInt::getFalse(M->getContext())});
+ ConstantInt::getFalse(M->getContext())}));
if (StoreAlignment)
cast<MemSetPatternInst>(NewCall)->setDestAlignment(*StoreAlignment);
NewCall->setAAMetadata(AATags);
@@ -2600,27 +2600,29 @@ bool LoopIdiomRecognize::recognizePopcount() {
return true;
}
-static CallInst *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
- const DebugLoc &DL) {
+static Value *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
+ const DebugLoc &DL) {
Value *Ops[] = {Val};
Type *Tys[] = {Val->getType()};
- CallInst *CI = IRBuilder.CreateIntrinsic(Intrinsic::ctpop, Tys, Ops);
- CI->setDebugLoc(DL);
+ Value *V = IRBuilder.CreateIntrinsic(Intrinsic::ctpop, Tys, Ops);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->setDebugLoc(DL);
- return CI;
+ return V;
}
-static CallInst *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
- const DebugLoc &DL, bool ZeroCheck,
- Intrinsic::ID IID) {
+static Value *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
+ const DebugLoc &DL, bool ZeroCheck,
+ Intrinsic::ID IID) {
Value *Ops[] = {Val, IRBuilder.getInt1(ZeroCheck)};
Type *Tys[] = {Val->getType()};
- CallInst *CI = IRBuilder.CreateIntrinsic(IID, Tys, Ops);
- CI->setDebugLoc(DL);
+ Value *V = IRBuilder.CreateIntrinsic(IID, Tys, Ops);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->setDebugLoc(DL);
- return CI;
+ return V;
}
/// Transform the following loop (Using CTLZ, CTTZ is similar):
@@ -3126,7 +3128,7 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
Value *Mask =
Builder.CreateOr(LowBitMask, BitMask, BitPos->getName() + ".mask");
Value *XMasked = Builder.CreateAnd(X, Mask, X->getName() + ".masked");
- CallInst *XMaskedNumLeadingZeros = Builder.CreateIntrinsic(
+ Value *XMaskedNumLeadingZeros = Builder.CreateIntrinsic(
IntrID, Ty, {XMasked, /*is_zero_poison=*/Builder.getTrue()},
/*FMFSource=*/nullptr, XMasked->getName() + ".numleadingzeros");
Value *XMaskedNumActiveBits = Builder.CreateSub(
@@ -3486,7 +3488,7 @@ bool LoopIdiomRecognize::recognizeShiftUntilZero() {
// Step 1: Compute the loop's final IV value / trip count.
- CallInst *ValNumLeadingZeros = Builder.CreateIntrinsic(
+ Value *ValNumLeadingZeros = Builder.CreateIntrinsic(
IntrID, Ty, {Val, /*is_zero_poison=*/Builder.getFalse()},
/*FMFSource=*/nullptr, Val->getName() + ".numleadingzeros");
Value *ValNumActiveBits = Builder.CreateSub(
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index be186ffbf7e42..37e65822bafe0 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1814,7 +1814,7 @@ static void AddAlignmentAssumptions(CallBase &CB, InlineFunctionInfo &IFI) {
if (getKnownAlignment(ArgVal, DL, &CB, AC, &DT) >= *Alignment)
continue;
- CallInst *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
+ Value *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
DL, ArgVal, Alignment->value());
AC->registerAssumption(cast<AssumeInst>(NewAsmp));
}
@@ -3171,8 +3171,8 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
// code with llvm.stacksave/llvm.stackrestore intrinsics.
if (InlinedFunctionInfo.ContainsDynamicAllocas) {
// Insert the llvm.stacksave.
- CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
- .CreateStackSave("savedstack");
+ Value *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
+ .CreateStackSave("savedstack");
// Insert a call to llvm.stackrestore before any return instructions in the
// inlined function.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index b2e45c3340b9f..d09d6bc57ee11 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2278,9 +2278,9 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
// Get the backedge taken count and truncate or extended to the AR type.
Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty);
- CallInst *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
- {AbsStep, TruncTripCount},
- /*FMFSource=*/nullptr, "mul");
+ Value *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
+ {AbsStep, TruncTripCount},
+ /*FMFSource=*/nullptr, "mul");
Value *MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
Value *OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow");
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 4b21b20beb895..aceec08ea2582 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1773,7 +1773,7 @@ static void hoistConditionalLoadsStores(
// extended for vector types in the future.
assert(!getLoadStoreType(I)->isVectorTy() && "not implemented");
auto *Op0 = I->getOperand(0);
- CallInst *MaskedLoadStore = nullptr;
+ Value *MaskedLoadStore = nullptr;
if (auto *LI = dyn_cast<LoadInst>(I)) {
// Handle Load.
auto *Ty = I->getType();
@@ -1816,7 +1816,8 @@ static void hoistConditionalLoadsStores(
// same. Keep it.
// !annotation: Not impact semantics. Keep it.
if (const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
- MaskedLoadStore->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
+ if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
+ CI->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
I->dropUBImplyingAttrsAndUnknownMetadata({LLVMContext::MD_annotation});
// FIXME: DIAssignID is not supported for masked store yet.
// (Verifier::visitDIAssignIDMetadata)
@@ -1824,7 +1825,8 @@ static void hoistConditionalLoadsStores(
I->eraseMetadataIf([](unsigned MDKind, MDNode *Node) {
return Node->getMetadataID() == Metadata::DIAssignIDKind;
});
- MaskedLoadStore->copyMetadata(*I);
+ if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
+ CI->copyMetadata(*I);
I->eraseFromParent();
}
}
@@ -5755,7 +5757,7 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
Value* Cond = BI->getCondition();
assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.");
- CallInst *Assumption;
+ Value *Assumption;
if (BI->getSuccessor(0) == BB) {
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
Builder.CreateBr(BI->getSuccessor(1));
@@ -8916,7 +8918,7 @@ static bool removeUndefIntroducingPredecessor(BasicBlock *BB,
// Preserve guarding condition in assume, because it might not be
// inferrable from any dominating condition.
Value *Cond = BI->getCondition();
- CallInst *Assumption;
+ Value *Assumption;
if (BI->getSuccessor(0) == BB)
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
else
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 1cbc8d0bef5ae..9d5b7711c9c35 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -895,7 +895,8 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
Align MemSetAlign =
CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne();
CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
- AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(0));
+ AttrBuilder ArgAttrs(CI->getContext(),
+ CI->getAttributes().getParamAttrs(0));
NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
CI->getContext(), 0, ArgAttrs));
copyFlags(*CI, NewCI);
@@ -2251,7 +2252,7 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l)) {
if (Pow->doesNotAccessMemory()) {
- CallInst *NewExp10 =
+ Value *NewExp10 =
B.CreateIntrinsic(Intrinsic::exp10, {Ty}, {Expo}, Pow, "exp10");
return copyFlags(*Pow, NewExp10);
}
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index b5437ad8705bf..ca1c76f047caa 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1148,8 +1148,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
assert(TTI.isLegalMaskedLoad(VecTy, Alignment, AS,
TTI::MaskKind::ConstantMask));
Value *Mask = createMaskForExtraElements(C, cast<FixedVectorType>(VecTy));
- VecInst = Builder.CreateMaskedLoad(
- VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
+ VecInst = cast<Instruction>(Builder.CreateMaskedLoad(
+ VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
} else {
// This can happen due to a chain of redundant loads.
// In this case, just use the element-type, and avoid ExtractElement.
@@ -1240,8 +1240,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
TTI::MaskKind::ConstantMask));
Value *Mask =
createMaskForExtraElements(C, cast<FixedVectorType>(Vec->getType()));
- VecInst = Builder.CreateMaskedStore(
- Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
+ VecInst = cast<Instruction>(Builder.CreateMaskedStore(
+ Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
} else {
// Chain is in offset order, so C[0] is the instr with the lowest offset,
// i.e. the root of the vector.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5210088ded294..be1987b8b7742 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23672,7 +23672,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
setInsertPointAfterBundle(E);
LoadInst *LI = cast<LoadInst>(VL0);
- Instruction *NewLI;
+ Value *NewLI;
FixedVectorType *StridedLoadTy = nullptr;
Value *PO = LI->getPointerOperand();
if (E->State == TreeEntry::Vectorize) {
@@ -23735,16 +23735,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
static_cast<int>(
DL->getTypeAllocSize(ScalarTy))));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
- auto *Inst = Builder.CreateIntrinsic(
+ auto *Intr = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_load,
{StridedLoadTy, PO->getType(), StrideTy},
{PO, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedLoadEC)),
Builder.getInt32(StridedLoadEC)});
- Inst->addParamAttr(
- /*ArgNo=*/0,
- Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
- NewLI = Inst;
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(
+ /*ArgNo=*/0,
+ Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
+ NewLI = Intr;
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state");
Value *VecPtr = vectorizeOperand(E, 0);
@@ -23794,7 +23795,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
VecValue = FinalShuffle(VecValue, E);
Value *Ptr = SI->getPointerOperand();
- Instruction *ST;
+ Value *ST;
if (E->State == TreeEntry::Vectorize) {
ST = Builder.CreateAlignedStore(VecValue, Ptr, SI->getAlign());
} else {
@@ -23824,16 +23825,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
StrideTy, static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
if (StridedStoreTy != VecTy)
VecValue = Builder.CreateBitOrPointerCast(VecValue, StridedStoreTy);
- auto *Inst = Builder.CreateIntrinsic(
+ auto *Intr = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_store,
{StridedStoreTy, Ptr->getType(), StrideTy},
{VecValue, Ptr, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedStoreEC)),
Builder.getInt32(StridedStoreEC)});
- Inst->addParamAttr(
- /*ArgNo=*/1,
- Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
- ST = Inst;
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(
+ /*ArgNo=*/1,
+ Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
+ ST = Intr;
}
Value *V = PropagateIRFlags(ST);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f78651cf8de6c..4946a42ddb1cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2303,10 +2303,11 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
else
assert(Opcode == Instruction::Add && "only add or sub supported for now");
- auto *HistogramInst = State.Builder.CreateIntrinsic(
+ auto *HistogramV = State.Builder.CreateIntrinsic(
Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
- applyMetadata(*HistogramInst);
+ if (auto *HistogramInst = dyn_cast<Instruction>(HistogramV))
+ applyMetadata(*HistogramInst);
}
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
@@ -4082,7 +4083,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
- CallInst *NewLI;
+ Value *NewLI;
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Addr = State.get(getAddr(), !CreateGather);
Value *Mask = nullptr;
@@ -4099,11 +4100,12 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,
{Addr, Mask, EVL}, nullptr, "vp.op.load");
}
- NewLI->addParamAttr(
- 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- applyMetadata(*NewLI);
- Instruction *Res = NewLI;
- State.set(this, Res);
+ if (auto *CI = dyn_cast<CallInst>(NewLI)) {
+ CI->addParamAttr(
+ 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
+ applyMetadata(*CI);
+ }
+ State.set(this, NewLI);
}
InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
@@ -4146,14 +4148,15 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
Value *StoredVal = State.get(StoredVPValue);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
- Instruction *NewSI = nullptr;
+ Value *NewSI = nullptr;
if (CreateScatter)
NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
else if (Mask)
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- applyMetadata(*NewSI);
+ if (auto *I = dyn_cast<Instruction>(NewSI))
+ applyMetadata(*I);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4170,7 +4173,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
- CallInst *NewSI = nullptr;
+ Value *NewSI = nullptr;
Value *StoredVal = State.get(StoredValue);
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Mask = nullptr;
@@ -4189,9 +4192,11 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
Intrinsic::vp_store,
{StoredVal, Addr, Mask, EVL});
}
- NewSI->addParamAttr(
- 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- applyMetadata(*NewSI);
+ if (auto *CI = dyn_cast<CallInst>(NewSI)) {
+ CI->addParamAttr(
+ 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
+ applyMetadata(*CI);
+ }
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
@@ -4356,7 +4361,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(MaskForGaps && "Mask for Gaps is required but it is null");
}
- Instruction *NewLoad;
+ Value *NewLoad;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
Value *PoisonVec = PoisonValue::get(VecTy);
@@ -4366,9 +4371,11 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
} else
NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
Group->getAlign(), "wide.vec");
- applyMetadata(*NewLoad);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewLoad);
+ if (auto *NewI = dyn_cast<Instruction>(NewLoad)) {
+ applyMetadata(*NewI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewI);
+ }
ArrayRef<VPRecipeValue *> VPDefs = definedValues();
if (VecTy->isScalableTy()) {
@@ -4461,18 +4468,20 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- Instruction *NewStoreInstr;
+ Value *NewStore;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
- NewStoreInstr = State.Builder.CreateMaskedStore(
- IVec, ResAddr, Group->getAlign(), GroupMask);
+ NewStore = State.Builder.CreateMaskedStore(IVec, ResAddr, Group->getAlign(),
+ GroupMask);
} else
- NewStoreInstr =
+ NewStore =
State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());
- applyMetadata(*NewStoreInstr);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewStoreInstr);
+ if (auto *I = dyn_cast<Instruction>(NewStore)) {
+ applyMetadata(*I);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(I);
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4540,15 +4549,16 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Vectorize the interleaved load group.
if (isa<LoadInst>(Instr)) {
- CallInst *NewLoad = State.Builder.CreateIntrinsic(
+ Value *NewLoad = State.Builder.CreateIntrinsic(
VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL}, nullptr,
"wide.vp.load");
- NewLoad->addParamAttr(0,
- Attribute::getWithAlignment(Ctx, Group->getAlign()));
+ if (auto *CI = dyn_cast<CallInst>(NewLoad)) {
+ CI->addParamAttr(0, Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*NewLoad);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewLoad);
+ applyMetadata(*CI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(CI);
+ }
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
@@ -4604,15 +4614,17 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- CallInst *NewStore =
+ Value *NewStore =
State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,
{IVec, ResAddr, GroupMask, InterleaveEVL});
- NewStore->addParamAttr(1,
- Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*NewStore);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewStore);
+ if (auto *CI = dyn_cast<CallInst>(NewStore)) {
+ CI->addParamAttr(1, Attribute::getWithAlignment(Ctx, Group->getAlign()));
+
+ applyMetadata(*CI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(CI);
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 99e45bdc8ee21..d49df0ccd0c6f 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1813,7 +1813,7 @@ bool VectorCombine::foldBinopOfReductions(Instruction &I) {
else
VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
- Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
+ Value *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
replaceValue(I, *Rdx);
return true;
}
@@ -4245,14 +4245,15 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (IsPartialReduction)
ReduceInput = Builder.CreateShuffleVector(FinalVecV, ExtractMask);
- CallInst *ReducedResult;
+ Value *ReducedResult;
if (IsFloatReduction) {
Value *Identity = ConstantExpr::getBinOpIdentity(
*CommonBinOp, ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
CommonFMF.noSignedZeros());
ReducedResult = Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy},
{Identity, ReduceInput});
- ReducedResult->setFastMathFlags(CommonFMF);
+ if (auto *ReducedI = dyn_cast<Instruction>(ReducedResult))
+ ReducedI->setFastMathFlags(CommonFMF);
} else {
ReducedResult =
Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy}, {ReduceInput});
@@ -5936,12 +5937,13 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
- CallInst *NewVP =
+ Value *NewVP =
Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
{II->getMemoryPointerParam(), NewMask, NewEVL});
// Preserve the original alignment.
- NewVP->addParamAttrs(
- 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
+ if (auto *NewCI = dyn_cast<CallInst>(NewVP))
+ NewCI->addParamAttrs(
+ 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
replaceValue(*Cast, *NewVP);
return true;
}
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index b4f1c97f03aca..1698856bb76e3 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -169,17 +169,19 @@ TEST_F(IRBuilderTest, IntrinsicMangling) {
CallInst *Call;
// Mangled return type, no arguments.
- Call = Builder.CreateIntrinsic(Int64Ty, Intrinsic::coro_size, {});
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Int64Ty, Intrinsic::coro_size, {}));
EXPECT_EQ(Call->getCalledFunction()->getName(), "llvm.coro.size.i64");
// Void return type, mangled argument type.
- Call =
- Builder.CreateIntrinsic(VoidTy, Intrinsic::set_loop_iterations, Int64Val);
+ Call = cast<CallInst>(Builder.CreateIntrinsic(
+ VoidTy, Intrinsic::set_loop_iterations, Int64Val));
EXPECT_EQ(Call->getCalledFunction()->getName(),
"llvm.set.loop.iterations.i64");
// Mangled return type and argument type.
- Call = Builder.CreateIntrinsic(Int64Ty, Intrinsic::lround, DoubleVal);
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Int64Ty, Intrinsic::lround, DoubleVal));
EXPECT_EQ(Call->getCalledFunction()->getName(), "llvm.lround.i64.f64");
}
@@ -199,8 +201,9 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
Args.push_back(UndefValue::get(PredTy));
Args.push_back(UndefValue::get(SrcVecTy));
- Call = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_fcvtzs_i32f16, Args,
- nullptr, "aarch64.sve.fcvtzs.i32f16");
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::aarch64_sve_fcvtzs_i32f16, Args,
+ nullptr, "aarch64.sve.fcvtzs.i32f16"));
FTy = Call->getFunctionType();
EXPECT_EQ(FTy->getReturnType(), DstVecTy);
for (unsigned i = 0; i != Args.size(); ++i)
@@ -218,8 +221,9 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
Args.push_back(UndefValue::get(PredTy));
Args.push_back(UndefValue::get(VecTy));
- Call = Builder.CreateIntrinsic(Intrinsic::masked_load, {VecTy, PtrToVecTy},
- Args, nullptr, "masked.load");
+ Call = cast<CallInst>(Builder.CreateIntrinsic(Intrinsic::masked_load,
+ {VecTy, PtrToVecTy}, Args,
+ nullptr, "masked.load"));
FTy = Call->getFunctionType();
EXPECT_EQ(FTy->getReturnType(), VecTy);
for (unsigned i = 0; i != Args.size(); ++i)
@@ -360,12 +364,12 @@ TEST_F(IRBuilderTest, ConstrainedFP) {
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::experimental_constrained_fmul);
-
+
V = Builder.CreateFDiv(V, V);
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::experimental_constrained_fdiv);
-
+
V = Builder.CreateFRem(V, V);
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
@@ -459,18 +463,18 @@ TEST_F(IRBuilderTest, ConstrainedFP) {
EXPECT_EQ(RoundingMode::Dynamic, CII->getRoundingMode());
// Now override the defaults.
- Call = Builder.CreateConstrainedFPBinOp(
- Intrinsic::experimental_constrained_fadd, V, V, nullptr, "", nullptr,
- RoundingMode::TowardNegative, fp::ebMayTrap);
+ Call = cast<CallInst>(Builder.CreateConstrainedFPBinOp(
+ Intrinsic::experimental_constrained_fadd, V, V, nullptr, "", nullptr,
+ RoundingMode::TowardNegative, fp::ebMayTrap));
CII = cast<ConstrainedFPIntrinsic>(Call);
EXPECT_EQ(CII->getIntrinsicID(), Intrinsic::experimental_constrained_fadd);
EXPECT_EQ(fp::ebMayTrap, CII->getExceptionBehavior());
EXPECT_EQ(RoundingMode::TowardNegative, CII->getRoundingMode());
// Same as previous test for CreateConstrainedFPIntrinsic
- Call = Builder.CreateConstrainedFPIntrinsic(
+ Call = cast<CallInst>(Builder.CreateConstrainedFPIntrinsic(
Intrinsic::experimental_constrained_fadd, {V->getType()}, {V, V}, nullptr,
- "", nullptr, RoundingMode::TowardNegative, fp::ebMayTrap);
+ "", nullptr, RoundingMode::TowardNegative, fp::ebMayTrap));
CII = cast<ConstrainedFPIntrinsic>(Call);
EXPECT_EQ(CII->getIntrinsicID(), Intrinsic::experimental_constrained_fadd);
EXPECT_EQ(fp::ebMayTrap, CII->getExceptionBehavior());
@@ -696,7 +700,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
ASSERT_TRUE(isa<Instruction>(F));
FDiv = cast<Instruction>(F);
EXPECT_FALSE(FDiv->hasAllowReciprocal());
-
+
// Try individual flags.
FMF.clear();
FMF.setAllowReciprocal();
@@ -755,7 +759,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
EXPECT_TRUE(FAdd->hasApproxFunc());
EXPECT_TRUE(FAdd->hasAllowContract());
EXPECT_FALSE(FAdd->hasAllowReassoc());
-
+
FMF.setAllowReassoc();
Builder.clearFastMathFlags();
Builder.setFastMathFlags(FMF);
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 87d922d22eaac..2853397628142 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -327,7 +327,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicScalar) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 10));
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 20));
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::umax);
@@ -345,7 +346,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicVector) {
Args.push_back(Constant::getNullValue(RetTy));
Args.push_back(Constant::getNullValue(RetTy));
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::umax);
@@ -366,7 +368,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicAddressSpace) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 3)); // locality
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1)); // cache type
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::prefetch, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::prefetch, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::prefetch);
@@ -394,8 +397,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicVarArg) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0)); // NumCallArgs
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0)); // Flags
- CallInst *CI = Builder.CreateIntrinsic(
- RetTy, Intrinsic::experimental_gc_statepoint, Args);
+ auto *CI = cast<CallInst>(Builder.CreateIntrinsic(
+ RetTy, Intrinsic::experimental_gc_statepoint, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::experimental_gc_statepoint);
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 8ceeacf132da1..67b9ad2585c9f 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -1114,8 +1114,8 @@ TEST(Local, CanReplaceOperandWithVariable) {
// immarg.
Type *PtrPtr = B.getPtrTy(0);
Value *Alloca = B.CreateAlloca(PtrPtr, (unsigned)0);
- CallInst *GCRoot = B.CreateIntrinsic(
- Intrinsic::gcroot, {Alloca, Constant::getNullValue(PtrPtr)});
+ auto *GCRoot = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::gcroot, {Alloca, Constant::getNullValue(PtrPtr)}));
EXPECT_TRUE(canReplaceOperandWithVariable(GCRoot, 0)); // Alloca
EXPECT_FALSE(canReplaceOperandWithVariable(GCRoot, 1));
EXPECT_FALSE(canReplaceOperandWithVariable(GCRoot, 2));
>From 442e28185c111fa3b52e23cbf7d868c351ad071f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 9 Jun 2026 22:22:55 +0100
Subject: [PATCH 2/3] [IRBuilder] Load-store-assume don't fold
---
clang/lib/CodeGen/CGHLSLBuiltins.cpp | 2 +-
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 3 +-
llvm/include/llvm/IR/IRBuilder.h | 90 +++++-----
llvm/lib/IR/IRBuilder.cpp | 151 ++++++++--------
.../AArch64/AArch64TargetTransformInfo.cpp | 12 +-
.../Target/ARM/MVEGatherScatterLowering.cpp | 170 +++++++++---------
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 15 +-
.../Target/Hexagon/HexagonVectorCombine.cpp | 11 +-
.../Target/X86/X86InstCombineIntrinsic.cpp | 2 +-
llvm/lib/Transforms/Utils/InlineFunction.cpp | 6 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 12 +-
.../Vectorize/LoadStoreVectorizer.cpp | 8 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 ++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 118 ++++++------
mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 4 +-
15 files changed, 306 insertions(+), 324 deletions(-)
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index e9511f5729f15..a534eb4cf20a0 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -124,7 +124,7 @@ static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
- CallInst *CI = CGF->Builder.CreateIntrinsic(
+ Value *CI = CGF->Builder.CreateIntrinsic(
RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble");
LowBits = CGF->Builder.CreateExtractValue(CI, 0);
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 21f32b12c4fd1..af7ea04027c48 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -341,7 +341,8 @@ emitAMDGCNImageOverloadedReturnType(clang::CodeGen::CodeGenFunction &CGF,
}
llvm::Type *RetTy = IsImageStore ? CGF.VoidTy : CGF.ConvertType(E->getType());
- llvm::CallInst *Call = CGF.Builder.CreateIntrinsic(RetTy, IntrinsicID, Args);
+ auto *Call = cast<llvm::CallInst>(
+ CGF.Builder.CreateIntrinsic(RetTy, IntrinsicID, Args));
return Call;
}
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 967a3fd8941d6..694b4961453a4 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -857,34 +857,36 @@ class IRBuilderBase {
LLVM_ABI CallInst *CreateThreadLocalAddress(Value *Ptr);
/// Create a call to Masked Load intrinsic
- LLVM_ABI Value *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Store intrinsic
- LLVM_ABI Value *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
- Value *Mask);
+ LLVM_ABI CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+ Value *Mask);
/// Create a call to Masked Gather intrinsic
- LLVM_ABI Value *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Scatter intrinsic
- LLVM_ABI Value *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
- Value *Mask = nullptr);
+ LLVM_ABI CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs,
+ Align Alignment,
+ Value *Mask = nullptr);
/// Create a call to Masked Expand Load intrinsic
- LLVM_ABI Value *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
+ MaybeAlign Align,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Compress Store intrinsic
- LLVM_ABI Value *CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align,
- Value *Mask = nullptr);
+ LLVM_ABI CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align,
+ Value *Mask = nullptr);
/// Return an all true boolean vector (mask) with \p NumElts lanes.
Value *getAllOnesMask(ElementCount NumElts) {
@@ -894,15 +896,15 @@ class IRBuilderBase {
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided condition will be true.
- LLVM_ABI Value *CreateAssumption(Value *Cond);
+ LLVM_ABI CallInst *CreateAssumption(Value *Cond);
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided operand bundles hold.
- LLVM_ABI Value *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
+ LLVM_ABI CallInst *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
/// Create a llvm.experimental.noalias.scope.decl intrinsic call.
- LLVM_ABI Value *CreateNoAliasScopeDeclaration(Value *Scope);
- Value *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
+ LLVM_ABI Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
+ Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
return CreateNoAliasScopeDeclaration(
MetadataAsValue::get(Context, ScopeTag));
}
@@ -1158,16 +1160,17 @@ class IRBuilderBase {
}
/// Create a call to llvm.stacksave
- Value *CreateStackSave(const Twine &Name = "") {
+ CallInst *CreateStackSave(const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
- return CreateIntrinsic(Intrinsic::stacksave, {DL.getAllocaPtrType(Context)},
- {}, nullptr, Name);
+ return cast<CallInst>(CreateIntrinsic(Intrinsic::stacksave,
+ {DL.getAllocaPtrType(Context)}, {},
+ nullptr, Name));
}
/// Create a call to llvm.stackrestore
- Value *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
- return CreateIntrinsic(Intrinsic::stackrestore, {Ptr->getType()}, {Ptr},
- nullptr, Name);
+ CallInst *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
+ return cast<CallInst>(CreateIntrinsic(
+ Intrinsic::stackrestore, {Ptr->getType()}, {Ptr}, nullptr, Name));
}
/// Create a call to llvm.experimental_cttz_elts
@@ -1181,9 +1184,9 @@ class IRBuilderBase {
private:
/// Create a call to a masked intrinsic with given Id.
- Value *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name = "");
+ CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name = "");
//===--------------------------------------------------------------------===//
// Instruction creation methods: Terminators
@@ -2808,8 +2811,9 @@ class IRBuilderBase {
/// Helper function that creates an assume intrinsic call that
/// represents an alignment assumption on the provided pointer \p PtrValue
/// with offset \p OffsetValue and alignment value \p AlignValue.
- Value *CreateAlignmentAssumptionHelper(const DataLayout &DL, Value *PtrValue,
- Value *AlignValue, Value *OffsetValue);
+ CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
+ Value *PtrValue, Value *AlignValue,
+ Value *OffsetValue);
public:
/// Create an assume intrinsic call that represents an alignment
@@ -2818,9 +2822,10 @@ class IRBuilderBase {
/// An optional offset can be provided, and if it is provided, the offset
/// must be subtracted from the provided pointer to get the pointer with the
/// specified alignment.
- LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue, uint64_t Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ uint64_t Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents an alignment
/// assumption on the provided pointer.
@@ -2831,18 +2836,19 @@ class IRBuilderBase {
///
/// This overload handles the condition where the Alignment is dependent
/// on an existing value rather than a static value.
- LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue, Value *Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ Value *Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents a dereferencable
/// assumption on the provided pointer.
- LLVM_ABI Value *CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue);
+ LLVM_ABI CallInst *CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue);
/// Create an assume intrinsic call that represents a nonnull assumption on
/// the provided pointer.
- LLVM_ABI Value *CreateNonnullAssumption(Value *PtrValue);
+ LLVM_ABI CallInst *CreateNonnullAssumption(Value *PtrValue);
};
/// This provides a uniform API for creating instructions and inserting
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 8f7e25fa58982..3f7fb19219793 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -546,21 +546,24 @@ CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
return CI;
}
-Value *IRBuilderBase::CreateAssumption(Value *Cond) {
+CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
assert(Cond->getType() == getInt1Ty() &&
"an assumption condition must be of type i1");
- return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, {Cond});
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, {Cond}));
}
-Value *IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
+CallInst *
+IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
Value *Args[] = {ConstantInt::getTrue(getContext())};
- return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, Args,
- /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
+ return cast<CallInst>(
+ CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, Args,
+ /*FMFSource=*/nullptr, /*Name=*/"", OpBundles));
}
-Value *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
- return CreateIntrinsic(Intrinsic::experimental_noalias_scope_decl, {},
- {Scope});
+Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
+ return cast<Instruction>(
+ CreateIntrinsic(Intrinsic::experimental_noalias_scope_decl, {}, {Scope}));
}
/// Create a call to a Masked Load intrinsic.
@@ -572,9 +575,9 @@ Value *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-Value *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru,
- const Twine &Name) {
+CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru,
+ const Twine &Name) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
@@ -582,12 +585,10 @@ Value *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
PassThru = PoisonValue::get(Ty);
Type *OverloadedTypes[] = { Ty, PtrTy };
Value *Ops[] = {Ptr, Mask, PassThru};
- Value *V =
+ CallInst *CI =
CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name);
- if (auto *CI = dyn_cast<CallInst>(V))
- CI->addParamAttr(0,
- Attribute::getWithAlignment(CI->getContext(), Alignment));
- return V;
+ CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return CI;
}
/// Create a call to a Masked Store intrinsic.
@@ -596,30 +597,28 @@ Value *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
/// \p Alignment - alignment of the destination location
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-Value *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
- Value *Mask) {
+CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
+ Align Alignment, Value *Mask) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *OverloadedTypes[] = { DataTy, PtrTy };
Value *Ops[] = {Val, Ptr, Mask};
- Value *V =
+ CallInst *CI =
CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
- if (auto *CI = dyn_cast<CallInst>(V))
- CI->addParamAttr(1,
- Attribute::getWithAlignment(CI->getContext(), Alignment));
- return V;
+ CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return CI;
}
/// Create a call to a Masked intrinsic, with given intrinsic Id,
/// an array of operands - Ops, and an array of overloaded types -
/// OverloadedTypes.
-Value *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
- ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name) {
- return CreateIntrinsic(Id, OverloadedTypes, Ops, {}, Name);
+CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
+ ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name) {
+ return cast<CallInst>(CreateIntrinsic(Id, OverloadedTypes, Ops, {}, Name));
}
/// Create a call to a Masked Gather intrinsic.
@@ -631,9 +630,10 @@ Value *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-Value *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
- Value *Mask, Value *PassThru,
- const Twine &Name) {
+CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
+ Align Alignment, Value *Mask,
+ Value *PassThru,
+ const Twine &Name) {
auto *VecTy = cast<VectorType>(Ty);
ElementCount NumElts = VecTy->getElementCount();
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
@@ -650,12 +650,10 @@ Value *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- Value *V = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
- OverloadedTypes, Name);
- if (auto *CI = dyn_cast<CallInst>(V))
- CI->addParamAttr(0,
- Attribute::getWithAlignment(CI->getContext(), Alignment));
- return V;
+ CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
+ OverloadedTypes, Name);
+ CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return CI;
}
/// Create a call to a Masked Scatter intrinsic.
@@ -665,8 +663,8 @@ Value *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
/// \p Align - alignment for one element
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-Value *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
- Align Alignment, Value *Mask) {
+CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
+ Align Alignment, Value *Mask) {
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
auto *DataTy = cast<VectorType>(Data->getType());
ElementCount NumElts = PtrsTy->getElementCount();
@@ -679,12 +677,10 @@ Value *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- Value *V =
+ CallInst *CI =
CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
- if (auto *CI = dyn_cast<CallInst>(V))
- CI->addParamAttr(1,
- Attribute::getWithAlignment(CI->getContext(), Alignment));
- return V;
+ CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return CI;
}
/// Create a call to Masked Expand Load intrinsic
@@ -696,10 +692,10 @@ Value *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-Value *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
- MaybeAlign Align, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
+ MaybeAlign Align, Value *Mask,
+ Value *PassThru,
+ const Twine &Name) {
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
if (!PassThru)
@@ -707,13 +703,11 @@ Value *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {Ty, PtrTy};
Value *Ops[] = {Ptr, Mask, PassThru};
- Value *V = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
- OverloadedTypes, Name);
- if (auto *CI = dyn_cast<CallInst>(V))
- if (Align)
- CI->addParamAttr(0,
- Attribute::getWithAlignment(CI->getContext(), *Align));
- return V;
+ CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
+ OverloadedTypes, Name);
+ if (Align)
+ CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *Align));
+ return CI;
}
/// Create a call to Masked Compress Store intrinsic
@@ -722,21 +716,20 @@ Value *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
/// \p Align - alignment of \p Ptr
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-Value *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align, Value *Mask) {
+CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align,
+ Value *Mask) {
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {DataTy, PtrTy};
Value *Ops[] = {Val, Ptr, Mask};
- Value *V = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
- OverloadedTypes);
- if (auto *CI = dyn_cast<CallInst>(V))
- if (Align)
- CI->addParamAttr(1,
- Attribute::getWithAlignment(CI->getContext(), *Align));
- return V;
+ CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
+ OverloadedTypes);
+ if (Align)
+ CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), *Align));
+ return CI;
}
template <typename T0>
@@ -1378,10 +1371,10 @@ Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
{FPNum, TestV});
}
-Value *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
- Value *PtrValue,
- Value *AlignValue,
- Value *OffsetValue) {
+CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
+ Value *PtrValue,
+ Value *AlignValue,
+ Value *OffsetValue) {
SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
if (OffsetValue)
Vals.push_back(OffsetValue);
@@ -1389,10 +1382,10 @@ Value *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
return CreateAssumption({AlignOpB});
}
-Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- uint64_t Alignment,
- Value *OffsetValue) {
+CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ uint64_t Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
assert(Alignment != 0 && "Invalid Alignment");
@@ -1402,17 +1395,17 @@ Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
}
-Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- Value *Alignment,
- Value *OffsetValue) {
+CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ Value *Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
}
-Value *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue) {
+CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a deferenceable assumption on a non-pointer?");
SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
@@ -1420,7 +1413,7 @@ Value *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
return CreateAssumption({DereferenceableOpB});
}
-Value *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
+CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a nonnull assumption on a non-pointer?");
return CreateAssumption(OperandBundleDef("nonnull", PtrValue));
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 14b151180f6a9..8e2a450e0697e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2457,11 +2457,10 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.replaceInstUsesWith(II, Load);
}
- Value *MaskedLoad =
+ CallInst *MaskedLoad =
IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
Pred, ConstantAggregateZero::get(VecTy));
- if (auto *NewI = dyn_cast<Instruction>(MaskedLoad))
- NewI->copyMetadata(II);
+ MaskedLoad->copyMetadata(II);
return IC.replaceInstUsesWith(II, MaskedLoad);
}
@@ -2477,10 +2476,9 @@ instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.eraseInstFromFunction(II);
}
- Value *MaskedStore = IC.Builder.CreateMaskedStore(
+ CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
VecOp, PtrOp, PtrOp->getPointerAlignment(DL), Pred);
- if (auto *NewI = dyn_cast<Instruction>(MaskedStore))
- NewI->copyMetadata(II);
+ MaskedStore->copyMetadata(II);
return IC.eraseInstFromFunction(II);
}
@@ -2722,7 +2720,7 @@ instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
BasePtr, IndexBase);
- Value *MaskedLoad =
+ CallInst *MaskedLoad =
IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
MaskedLoad->takeName(&II);
return IC.replaceInstUsesWith(II, MaskedLoad);
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 5798f11bda16a..69c27def9c1ee 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -103,41 +103,43 @@ class MVEGatherScatterLowering : public FunctionPass {
// the other summand.
std::pair<Value *, int64_t> getVarAndConst(Value *Inst, int TypeScale);
- Value *lowerGather(IntrinsicInst *I);
+ Instruction *lowerGather(IntrinsicInst *I);
// Create a gather from a base + vector of offsets
- Value *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
- Instruction *&Root, IRBuilder<> &Builder);
+ Instruction *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
+ Instruction *&Root,
+ IRBuilder<> &Builder);
// Create a gather from a vector of pointers
- Value *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder, int64_t Increment = 0);
+ Instruction *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// Create an incrementing gather from a vector of pointers
- Value *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Instruction *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
- Value *lowerScatter(IntrinsicInst *I);
+ Instruction *lowerScatter(IntrinsicInst *I);
// Create a scatter to a base + vector of offsets
- Value *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
- IRBuilder<> &Builder);
+ Instruction *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
+ IRBuilder<> &Builder);
// Create a scatter to a vector of pointers
- Value *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Instruction *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// Create an incrementing scatter from a vector of pointers
- Value *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Instruction *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// QI gathers and scatters can increment their offsets on their own if
// the increment is a constant value (digit)
- Value *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder);
+ Instruction *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder);
// QI gathers/scatters can increment their offsets on their own if the
// increment is a constant value (digit) - this creates a writeback QI
// gather/scatter
- Value *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
- Value *Ptr, unsigned TypeScale,
- IRBuilder<> &Builder);
+ Instruction *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
+ Value *Ptr, unsigned TypeScale,
+ IRBuilder<> &Builder);
// Optimise the base and offsets of the given address
bool optimiseAddress(Value *Address, BasicBlock *BB, LoopInfo *LI);
@@ -395,7 +397,7 @@ MVEGatherScatterLowering::getVarAndConst(Value *Inst, int TypeScale) {
return std::pair<Value *, int64_t>(Summand, Immediate);
}
-Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
+Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked gathers: checking transform preconditions\n"
<< *I << "\n");
@@ -421,7 +423,7 @@ Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
Instruction *Root = I;
- Value *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Instruction *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Load)
Load = tryCreateMaskedGatherOffset(I, Ptr, Root, Builder);
if (!Load)
@@ -448,10 +450,8 @@ Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
return Load;
}
-Value *MVEGatherScatterLowering::tryCreateMaskedGatherBase(IntrinsicInst *I,
- Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment) {
+Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
+ IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
LLVM_DEBUG(dbgs() << "masked gathers: loading from vector of pointers\n");
@@ -460,17 +460,16 @@ Value *MVEGatherScatterLowering::tryCreateMaskedGatherBase(IntrinsicInst *I,
return nullptr;
Value *Mask = I->getArgOperand(1);
if (match(Mask, m_One()))
- return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base,
- {Ty, Ptr->getType()},
- {Ptr, Builder.getInt32(Increment)});
- else
- return Builder.CreateIntrinsic(
- Intrinsic::arm_mve_vldr_gather_base_predicated,
- {Ty, Ptr->getType(), Mask->getType()},
- {Ptr, Builder.getInt32(Increment), Mask});
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::arm_mve_vldr_gather_base, {Ty, Ptr->getType()},
+ {Ptr, Builder.getInt32(Increment)}));
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_predicated,
+ {Ty, Ptr->getType(), Mask->getType()},
+ {Ptr, Builder.getInt32(Increment), Mask}));
}
-Value *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
+Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
@@ -481,17 +480,16 @@ Value *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
return nullptr;
Value *Mask = I->getArgOperand(1);
if (match(Mask, m_One()))
- return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_wb,
- {Ty, Ptr->getType()},
- {Ptr, Builder.getInt32(Increment)});
- else
- return Builder.CreateIntrinsic(
- Intrinsic::arm_mve_vldr_gather_base_wb_predicated,
- {Ty, Ptr->getType(), Mask->getType()},
- {Ptr, Builder.getInt32(Increment), Mask});
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::arm_mve_vldr_gather_base_wb, {Ty, Ptr->getType()},
+ {Ptr, Builder.getInt32(Increment)}));
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_wb_predicated,
+ {Ty, Ptr->getType(), Mask->getType()},
+ {Ptr, Builder.getInt32(Increment), Mask}));
}
-Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
+Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
IntrinsicInst *I, Value *Ptr, Instruction *&Root, IRBuilder<> &Builder) {
using namespace PatternMatch;
@@ -553,19 +551,19 @@ Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
Root = Extend;
Value *Mask = I->getArgOperand(1);
- Value *Load = nullptr;
+ Instruction *Load = nullptr;
if (!match(Mask, m_One()))
- Load = Builder.CreateIntrinsic(
+ Load = cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::arm_mve_vldr_gather_offset_predicated,
{ResultTy, BasePtr->getType(), Offsets->getType(), Mask->getType()},
{BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
- Builder.getInt32(Scale), Builder.getInt32(Unsigned), Mask});
+ Builder.getInt32(Scale), Builder.getInt32(Unsigned), Mask}));
else
- Load = Builder.CreateIntrinsic(
+ Load = cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::arm_mve_vldr_gather_offset,
{ResultTy, BasePtr->getType(), Offsets->getType()},
{BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
- Builder.getInt32(Scale), Builder.getInt32(Unsigned)});
+ Builder.getInt32(Scale), Builder.getInt32(Unsigned)}));
if (TruncResult) {
Load = TruncInst::Create(Instruction::Trunc, Load, MemoryTy);
@@ -574,7 +572,7 @@ Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
return Load;
}
-Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
+Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked scatters: checking transform preconditions\n"
<< *I << "\n");
@@ -598,7 +596,7 @@ Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
Builder.SetInsertPoint(I);
Builder.SetCurrentDebugLocation(I->getDebugLoc());
- Value *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Instruction *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Store)
Store = tryCreateMaskedScatterOffset(I, Ptr, Builder);
if (!Store)
@@ -612,7 +610,7 @@ Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
return Store;
}
-Value *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
+Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -626,17 +624,17 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
// int_arm_mve_vstr_scatter_base(_predicated) addr, offset, data(, mask)
LLVM_DEBUG(dbgs() << "masked scatters: storing to a vector of pointers\n");
if (match(Mask, m_One()))
- return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base,
- {Ptr->getType(), Input->getType()},
- {Ptr, Builder.getInt32(Increment), Input});
- else
- return Builder.CreateIntrinsic(
- Intrinsic::arm_mve_vstr_scatter_base_predicated,
- {Ptr->getType(), Input->getType(), Mask->getType()},
- {Ptr, Builder.getInt32(Increment), Input, Mask});
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base,
+ {Ptr->getType(), Input->getType()},
+ {Ptr, Builder.getInt32(Increment), Input}));
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::arm_mve_vstr_scatter_base_predicated,
+ {Ptr->getType(), Input->getType(), Mask->getType()},
+ {Ptr, Builder.getInt32(Increment), Input, Mask}));
}
-Value *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
+Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -648,17 +646,17 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
return nullptr;
Value *Mask = I->getArgOperand(2);
if (match(Mask, m_One()))
- return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base_wb,
- {Ptr->getType(), Input->getType()},
- {Ptr, Builder.getInt32(Increment), Input});
- else
- return Builder.CreateIntrinsic(
- Intrinsic::arm_mve_vstr_scatter_base_wb_predicated,
- {Ptr->getType(), Input->getType(), Mask->getType()},
- {Ptr, Builder.getInt32(Increment), Input, Mask});
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base_wb,
+ {Ptr->getType(), Input->getType()},
+ {Ptr, Builder.getInt32(Increment), Input}));
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::arm_mve_vstr_scatter_base_wb_predicated,
+ {Ptr->getType(), Input->getType(), Mask->getType()},
+ {Ptr, Builder.getInt32(Increment), Input, Mask}));
}
-Value *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
+Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -707,23 +705,23 @@ Value *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
if (ExtendInput)
Input = Builder.CreateZExt(Input, InputTy);
if (!match(Mask, m_One()))
- return Builder.CreateIntrinsic(
+ return cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::arm_mve_vstr_scatter_offset_predicated,
{BasePtr->getType(), Offsets->getType(), Input->getType(),
Mask->getType()},
{BasePtr, Offsets, Input,
Builder.getInt32(MemoryTy->getScalarSizeInBits()),
- Builder.getInt32(Scale), Mask});
+ Builder.getInt32(Scale), Mask}));
else
- return Builder.CreateIntrinsic(
+ return cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::arm_mve_vstr_scatter_offset,
{BasePtr->getType(), Offsets->getType(), Input->getType()},
{BasePtr, Offsets, Input,
Builder.getInt32(MemoryTy->getScalarSizeInBits()),
- Builder.getInt32(Scale)});
+ Builder.getInt32(Scale)}));
}
-Value *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
+Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
FixedVectorType *Ty;
if (I->getIntrinsicID() == Intrinsic::masked_gather)
@@ -798,7 +796,7 @@ Value *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
return tryCreateMaskedScatterBase(I, OffsetsIncoming, Builder, Immediate);
}
-Value *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
+Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
IntrinsicInst *I, Value *BasePtr, Value *Offsets, unsigned TypeScale,
IRBuilder<> &Builder) {
// Check whether this gather's offset is incremented by a constant - if so,
@@ -860,8 +858,8 @@ Value *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
Builder.SetInsertPoint(I);
- Value *EndResult;
- Value *NewInduction;
+ Instruction *EndResult;
+ Instruction *NewInduction;
if (I->getIntrinsicID() == Intrinsic::masked_gather) {
// Build the incrementing gather
Value *Load = tryCreateMaskedGatherBaseWB(I, Phi, Builder, Immediate);
@@ -1281,22 +1279,22 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) {
}
}
for (IntrinsicInst *I : Gathers) {
- Value *L = lowerGather(I);
- if (!isa_and_nonnull<Instruction>(L))
+ Instruction *L = lowerGather(I);
+ if (L == nullptr)
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(cast<Instruction>(L)->getParent());
+ SimplifyInstructionsInBlock(L->getParent());
Changed = true;
}
for (IntrinsicInst *I : Scatters) {
- Value *S = lowerScatter(I);
- if (!isa_and_nonnull<Instruction>(S))
+ Instruction *S = lowerScatter(I);
+ if (S == nullptr)
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(cast<Instruction>(S)->getParent());
+ SimplifyInstructionsInBlock(S->getParent());
Changed = true;
}
return Changed;
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index 4178bac9367cb..b12985d81fa75 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -488,8 +488,9 @@ static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
return aspaceWrapValue(Cache, CI->getFunction(), P);
}
-static Value *aspaceMemSet(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Instruction *aspaceMemSet(Intrinsic::ID ID,
+ DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -514,8 +515,9 @@ static Value *aspaceMemSet(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
MI->getAAMetadata());
}
-static Value *aspaceMemCpy(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Instruction *aspaceMemCpy(Intrinsic::ID ID,
+ DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -538,7 +540,8 @@ static Value *aspaceMemCpy(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
IsVolatile, MI->getAAMetadata());
}
-static Value *aspaceMemMove(DenseMap<Value *, Value *> &Cache, CallInst *CI) {
+static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -617,7 +620,7 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
if (!IsSet && !IsCpy && !IsMove)
continue;
- Value *New;
+ Instruction *New;
if (IsSet)
New = aspaceMemSet(ID, CastsCache, CI);
else if (IsCpy)
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 110d3abc8f417..9e305f8039812 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -803,11 +803,10 @@ auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
if (HVC.isTrue(Mask))
return createSimpleLoad(Builder, ValTy, Ptr, EffA.value(), MDSources);
- Value *Load =
+ Instruction *Load =
Builder.CreateMaskedLoad(ValTy, Ptr, EffA, Mask, PassThru, "mld");
LLVM_DEBUG(dbgs() << "\t[Creating masked Load:] "; Load->dump());
- if (auto *LoadI = dyn_cast<Instruction>(Load))
- propagateMetadata(LoadI, MDSources);
+ propagateMetadata(Load, MDSources);
return Load;
}
@@ -872,9 +871,9 @@ auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
// Mask is unknown
if (!Predicate) {
- Value *Store = Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
- if (auto *StoreI = dyn_cast<Instruction>(Store))
- propagateMetadata(StoreI, MDSources);
+ Instruction *Store =
+ Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
+ propagateMetadata(Store, MDSources);
return Store;
}
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 7eab4deb9a14c..4999581489e82 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -68,7 +68,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
// intrinsic to the LLVM intrinsic to allow target-independent optimizations.
if (Value *BoolMask = getBoolVecFromMask(Mask, IC.getDataLayout())) {
// The pass-through vector for an x86 masked load is a zero vector.
- Value *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
+ CallInst *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
II.getType(), Ptr, Align(1), BoolMask, ZeroVec);
return IC.replaceInstUsesWith(II, NewMaskedLoad);
}
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 37e65822bafe0..be186ffbf7e42 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1814,7 +1814,7 @@ static void AddAlignmentAssumptions(CallBase &CB, InlineFunctionInfo &IFI) {
if (getKnownAlignment(ArgVal, DL, &CB, AC, &DT) >= *Alignment)
continue;
- Value *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
+ CallInst *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
DL, ArgVal, Alignment->value());
AC->registerAssumption(cast<AssumeInst>(NewAsmp));
}
@@ -3171,8 +3171,8 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
// code with llvm.stacksave/llvm.stackrestore intrinsics.
if (InlinedFunctionInfo.ContainsDynamicAllocas) {
// Insert the llvm.stacksave.
- Value *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
- .CreateStackSave("savedstack");
+ CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
+ .CreateStackSave("savedstack");
// Insert a call to llvm.stackrestore before any return instructions in the
// inlined function.
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index aceec08ea2582..4b21b20beb895 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1773,7 +1773,7 @@ static void hoistConditionalLoadsStores(
// extended for vector types in the future.
assert(!getLoadStoreType(I)->isVectorTy() && "not implemented");
auto *Op0 = I->getOperand(0);
- Value *MaskedLoadStore = nullptr;
+ CallInst *MaskedLoadStore = nullptr;
if (auto *LI = dyn_cast<LoadInst>(I)) {
// Handle Load.
auto *Ty = I->getType();
@@ -1816,8 +1816,7 @@ static void hoistConditionalLoadsStores(
// same. Keep it.
// !annotation: Not impact semantics. Keep it.
if (const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
- if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
- CI->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
+ MaskedLoadStore->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
I->dropUBImplyingAttrsAndUnknownMetadata({LLVMContext::MD_annotation});
// FIXME: DIAssignID is not supported for masked store yet.
// (Verifier::visitDIAssignIDMetadata)
@@ -1825,8 +1824,7 @@ static void hoistConditionalLoadsStores(
I->eraseMetadataIf([](unsigned MDKind, MDNode *Node) {
return Node->getMetadataID() == Metadata::DIAssignIDKind;
});
- if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
- CI->copyMetadata(*I);
+ MaskedLoadStore->copyMetadata(*I);
I->eraseFromParent();
}
}
@@ -5757,7 +5755,7 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
Value* Cond = BI->getCondition();
assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.");
- Value *Assumption;
+ CallInst *Assumption;
if (BI->getSuccessor(0) == BB) {
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
Builder.CreateBr(BI->getSuccessor(1));
@@ -8918,7 +8916,7 @@ static bool removeUndefIntroducingPredecessor(BasicBlock *BB,
// Preserve guarding condition in assume, because it might not be
// inferrable from any dominating condition.
Value *Cond = BI->getCondition();
- Value *Assumption;
+ CallInst *Assumption;
if (BI->getSuccessor(0) == BB)
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
else
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index ca1c76f047caa..b5437ad8705bf 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1148,8 +1148,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
assert(TTI.isLegalMaskedLoad(VecTy, Alignment, AS,
TTI::MaskKind::ConstantMask));
Value *Mask = createMaskForExtraElements(C, cast<FixedVectorType>(VecTy));
- VecInst = cast<Instruction>(Builder.CreateMaskedLoad(
- VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
+ VecInst = Builder.CreateMaskedLoad(
+ VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
} else {
// This can happen due to a chain of redundant loads.
// In this case, just use the element-type, and avoid ExtractElement.
@@ -1240,8 +1240,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
TTI::MaskKind::ConstantMask));
Value *Mask =
createMaskForExtraElements(C, cast<FixedVectorType>(Vec->getType()));
- VecInst = cast<Instruction>(Builder.CreateMaskedStore(
- Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
+ VecInst = Builder.CreateMaskedStore(
+ Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
} else {
// Chain is in offset order, so C[0] is the instr with the lowest offset,
// i.e. the root of the vector.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index be1987b8b7742..defb557ce2abb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23735,17 +23735,16 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
static_cast<int>(
DL->getTypeAllocSize(ScalarTy))));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
- auto *Intr = Builder.CreateIntrinsic(
+ auto *Inst = cast<CallInst>(Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_load,
{StridedLoadTy, PO->getType(), StrideTy},
{PO, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedLoadEC)),
- Builder.getInt32(StridedLoadEC)});
- if (auto *CI = dyn_cast<CallInst>(Intr))
- CI->addParamAttr(
- /*ArgNo=*/0,
- Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
- NewLI = Intr;
+ Builder.getInt32(StridedLoadEC)}));
+ Inst->addParamAttr(
+ /*ArgNo=*/0,
+ Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
+ NewLI = Inst;
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state");
Value *VecPtr = vectorizeOperand(E, 0);
@@ -23825,17 +23824,16 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
StrideTy, static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
if (StridedStoreTy != VecTy)
VecValue = Builder.CreateBitOrPointerCast(VecValue, StridedStoreTy);
- auto *Intr = Builder.CreateIntrinsic(
+ auto *Inst = cast<CallInst>(Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_store,
{StridedStoreTy, Ptr->getType(), StrideTy},
{VecValue, Ptr, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedStoreEC)),
- Builder.getInt32(StridedStoreEC)});
- if (auto *CI = dyn_cast<CallInst>(Intr))
- CI->addParamAttr(
- /*ArgNo=*/1,
- Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
- ST = Intr;
+ Builder.getInt32(StridedStoreEC)}));
+ Inst->addParamAttr(
+ /*ArgNo=*/1,
+ Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
+ ST = Inst;
}
Value *V = PropagateIRFlags(ST);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4946a42ddb1cc..c09885de8a637 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2303,11 +2303,10 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
else
assert(Opcode == Instruction::Add && "only add or sub supported for now");
- auto *HistogramV = State.Builder.CreateIntrinsic(
+ auto *HistogramInst = cast<Instruction>(State.Builder.CreateIntrinsic(
Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
- {Address, IncAmt, Mask});
- if (auto *HistogramInst = dyn_cast<Instruction>(HistogramV))
- applyMetadata(*HistogramInst);
+ {Address, IncAmt, Mask}));
+ applyMetadata(*HistogramInst);
}
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
@@ -4083,7 +4082,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
- Value *NewLI;
+ CallInst *NewLI;
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Addr = State.get(getAddr(), !CreateGather);
Value *Mask = nullptr;
@@ -4093,18 +4092,16 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
if (CreateGather) {
- NewLI =
- Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
- nullptr, "wide.masked.gather");
+ NewLI = cast<CallInst>(Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather,
+ {Addr, Mask, EVL}, nullptr,
+ "wide.masked.gather"));
} else {
- NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,
- {Addr, Mask, EVL}, nullptr, "vp.op.load");
- }
- if (auto *CI = dyn_cast<CallInst>(NewLI)) {
- CI->addParamAttr(
- 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- applyMetadata(*CI);
+ NewLI = cast<CallInst>(Builder.CreateIntrinsic(
+ DataTy, Intrinsic::vp_load, {Addr, Mask, EVL}, nullptr, "vp.op.load"));
}
+ NewLI->addParamAttr(
+ 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
+ applyMetadata(*NewLI);
State.set(this, NewLI);
}
@@ -4148,15 +4145,14 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
Value *StoredVal = State.get(StoredVPValue);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
- Value *NewSI = nullptr;
+ Instruction *NewSI = nullptr;
if (CreateScatter)
NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
else if (Mask)
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- if (auto *I = dyn_cast<Instruction>(NewSI))
- applyMetadata(*I);
+ applyMetadata(*NewSI);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4173,7 +4169,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
- Value *NewSI = nullptr;
+ CallInst *NewSI = nullptr;
Value *StoredVal = State.get(StoredValue);
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Mask = nullptr;
@@ -4184,19 +4180,17 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
Value *Addr = State.get(getAddr(), !CreateScatter);
if (CreateScatter) {
- NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
- Intrinsic::vp_scatter,
- {StoredVal, Addr, Mask, EVL});
+ NewSI = cast<CallInst>(Builder.CreateIntrinsic(
+ Type::getVoidTy(EVL->getContext()), Intrinsic::vp_scatter,
+ {StoredVal, Addr, Mask, EVL}));
} else {
- NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
- Intrinsic::vp_store,
- {StoredVal, Addr, Mask, EVL});
- }
- if (auto *CI = dyn_cast<CallInst>(NewSI)) {
- CI->addParamAttr(
- 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- applyMetadata(*CI);
+ NewSI = cast<CallInst>(Builder.CreateIntrinsic(
+ Type::getVoidTy(EVL->getContext()), Intrinsic::vp_store,
+ {StoredVal, Addr, Mask, EVL}));
}
+ NewSI->addParamAttr(
+ 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
+ applyMetadata(*NewSI);
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
@@ -4361,7 +4355,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(MaskForGaps && "Mask for Gaps is required but it is null");
}
- Value *NewLoad;
+ Instruction *NewLoad;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
Value *PoisonVec = PoisonValue::get(VecTy);
@@ -4371,11 +4365,9 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
} else
NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
Group->getAlign(), "wide.vec");
- if (auto *NewI = dyn_cast<Instruction>(NewLoad)) {
- applyMetadata(*NewI);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewI);
- }
+ applyMetadata(*NewLoad);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewLoad);
ArrayRef<VPRecipeValue *> VPDefs = definedValues();
if (VecTy->isScalableTy()) {
@@ -4383,10 +4375,10 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// so must use intrinsics to deinterleave.
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
- NewLoad = State.Builder.CreateIntrinsic(
+ NewLoad = cast<Instruction>(State.Builder.CreateIntrinsic(
Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
NewLoad->getType(), NewLoad,
- /*FMFSource=*/nullptr, "strided.vec");
+ /*FMFSource=*/nullptr, "strided.vec"));
}
auto CreateStridedVector = [&InterleaveFactor, &State,
@@ -4468,20 +4460,18 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- Value *NewStore;
+ Instruction *NewStoreInstr;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
- NewStore = State.Builder.CreateMaskedStore(IVec, ResAddr, Group->getAlign(),
- GroupMask);
+ NewStoreInstr = State.Builder.CreateMaskedStore(
+ IVec, ResAddr, Group->getAlign(), GroupMask);
} else
- NewStore =
+ NewStoreInstr =
State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());
- if (auto *I = dyn_cast<Instruction>(NewStore)) {
- applyMetadata(*I);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(I);
- }
+ applyMetadata(*NewStoreInstr);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewStoreInstr);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4549,23 +4539,22 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Vectorize the interleaved load group.
if (isa<LoadInst>(Instr)) {
- Value *NewLoad = State.Builder.CreateIntrinsic(
+ auto *NewLoad = cast<CallInst>(State.Builder.CreateIntrinsic(
VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL}, nullptr,
- "wide.vp.load");
- if (auto *CI = dyn_cast<CallInst>(NewLoad)) {
- CI->addParamAttr(0, Attribute::getWithAlignment(Ctx, Group->getAlign()));
+ "wide.vp.load"));
+ NewLoad->addParamAttr(0,
+ Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*CI);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(CI);
- }
+ applyMetadata(*NewLoad);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewLoad);
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
- NewLoad = State.Builder.CreateIntrinsic(
+ NewLoad = cast<CallInst>(State.Builder.CreateIntrinsic(
Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
NewLoad->getType(), NewLoad,
- /*FMFSource=*/nullptr, "strided.vec");
+ /*FMFSource=*/nullptr, "strided.vec"));
const DataLayout &DL = Instr->getDataLayout();
for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
@@ -4614,17 +4603,16 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- Value *NewStore =
+ auto *NewStore = cast<CallInst>(
State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,
- {IVec, ResAddr, GroupMask, InterleaveEVL});
+ {IVec, ResAddr, GroupMask, InterleaveEVL}));
- if (auto *CI = dyn_cast<CallInst>(NewStore)) {
- CI->addParamAttr(1, Attribute::getWithAlignment(Ctx, Group->getAlign()));
+ NewStore->addParamAttr(1,
+ Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*CI);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(CI);
- }
+ applyMetadata(*NewStore);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewStore);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 01d3e9c4a62ce..d978184db8449 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -923,13 +923,13 @@ void mlir::LLVM::detail::connectPHINodes(Region ®ion,
}
}
-llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
+llvm::Value *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
return builder.CreateIntrinsic(intrinsic, tys, args);
}
-llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
+llvm::Value *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
llvm::Type *retTy, ArrayRef<llvm::Value *> args) {
return builder.CreateIntrinsic(retTy, intrinsic, args);
>From 9d76c151117a27e6e3181f86badbd0d591471b03 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 10 Jun 2026 07:45:54 +0100
Subject: [PATCH 3/3] [clang,mlir] Attempt to fix build
---
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 2 +-
clang/lib/CodeGen/TargetBuiltins/SPIR.cpp | 4 ++--
mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 10 ++++++----
3 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index af7ea04027c48..b2818f7553c8b 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -464,7 +464,7 @@ static Value *GetAMDGPUPredicate(CodeGenFunction &CGF, Twine Name) {
MDNode *Predicate = MDNode::get(Ctx, MDString::get(Ctx, Name.str()));
std::vector<Value *> Args = {SpecId, ConstantInt::getFalse(Ctx),
MetadataAsValue::get(Ctx, Predicate)};
- CallInst *Call = CGF.Builder.CreateIntrinsic(
+ Value *Call = CGF.Builder.CreateIntrinsic(
Intrinsic::spv_named_boolean_spec_constant, Args);
return Call;
diff --git a/clang/lib/CodeGen/TargetBuiltins/SPIR.cpp b/clang/lib/CodeGen/TargetBuiltins/SPIR.cpp
index ff7b5fefedd19..41de7ee2522a6 100644
--- a/clang/lib/CodeGen/TargetBuiltins/SPIR.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/SPIR.cpp
@@ -103,9 +103,9 @@ Value *CodeGenFunction::EmitSPIRVBuiltinExpr(unsigned BuiltinID,
llvm::Type *Res = getTypes().ConvertType(E->getType());
assert(Res->isPointerTy() &&
"GenericCastToPtrExplicit doesn't return a pointer");
- llvm::CallInst *Call = Builder.CreateIntrinsic(
+ llvm::CallInst *Call = cast<llvm::CallInst>(Builder.CreateIntrinsic(
/*ReturnType=*/Res, Intrinsic::spv_generic_cast_to_ptr_explicit,
- ArrayRef<Value *>{Ptr}, nullptr, "spv.generic_cast");
+ ArrayRef<Value *>{Ptr}, nullptr, "spv.generic_cast"));
Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
return Call;
}
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index d978184db8449..72f5ef4ecfb57 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -923,16 +923,18 @@ void mlir::LLVM::detail::connectPHINodes(Region ®ion,
}
}
-llvm::Value *mlir::LLVM::detail::createIntrinsicCall(
+llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
- return builder.CreateIntrinsic(intrinsic, tys, args);
+ return llvm::cast<llvm::CallInst>(
+ builder.CreateIntrinsic(intrinsic, tys, args));
}
-llvm::Value *mlir::LLVM::detail::createIntrinsicCall(
+llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
llvm::Type *retTy, ArrayRef<llvm::Value *> args) {
- return builder.CreateIntrinsic(retTy, intrinsic, args);
+ return llvm::cast<llvm::CallInst>(
+ builder.CreateIntrinsic(retTy, intrinsic, args));
}
llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
More information about the cfe-commits
mailing list