[llvm] [IRBuilder] Refactor for intrinsics const-folding (NFC) (PR #202738)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 12:02:06 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/202738
>From 8d2b728a5c09b01eb12bd3303e27220d993d5a9c Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 9 Jun 2026 15:07:07 +0100
Subject: [PATCH] [IRBuilder] Refactor for intrinsics const-folding (NFC)
In preparation to const-fold intrinsic calls, refactor the IRBuilder
API, generalizing it to return possibly constant-folded values.
---
llvm/include/llvm/IR/IRBuilder.h | 299 ++++++------
llvm/lib/CodeGen/InlineAsmPrepare.cpp | 4 +-
llvm/lib/CodeGen/SafeStack.cpp | 5 +-
llvm/lib/IR/AutoUpgrade.cpp | 15 +-
llvm/lib/IR/IRBuilder.cpp | 453 ++++++++++--------
.../Target/AArch64/AArch64ISelLowering.cpp | 9 +-
.../Target/AArch64/AArch64StackTagging.cpp | 8 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 14 +-
llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp | 4 +-
.../AMDGPU/AMDGPUAsanInstrumentation.cpp | 3 +-
.../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 4 +-
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 3 +-
.../AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp | 3 +-
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 36 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 8 +-
.../Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 52 +-
.../AMDGPU/AMDGPULowerKernelArguments.cpp | 4 +-
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 22 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +-
llvm/lib/Target/ARM/ARMISelLowering.cpp | 17 +-
.../Target/ARM/MVEGatherScatterLowering.cpp | 92 ++--
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 15 +-
.../Hexagon/HexagonLoopIdiomRecognition.cpp | 5 +-
.../Target/Hexagon/HexagonVectorCombine.cpp | 13 +-
llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp | 13 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 12 +-
.../Target/RISCV/RISCVInterleavedAccess.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 147 +++---
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 2 +-
.../Target/SPIRV/SPIRVLegalizePointerCast.cpp | 6 +-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 10 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 +-
llvm/lib/Target/SystemZ/SystemZTDC.cpp | 2 +-
.../Target/X86/X86InstCombineIntrinsic.cpp | 2 +-
llvm/lib/Target/X86/X86LowerAMXType.cpp | 13 +-
llvm/lib/Transforms/IPO/FatLTOCleanup.cpp | 2 +-
.../InstCombine/InstCombineAndOrXor.cpp | 2 +-
.../InstCombine/InstCombineCalls.cpp | 5 +-
.../InstCombine/InstCombineCompares.cpp | 4 +-
.../InstCombineLoadStoreAlloca.cpp | 9 +-
.../InstCombine/InstCombineMulDivRem.cpp | 10 +-
.../InstCombine/InstCombineSelect.cpp | 5 +-
.../InstCombine/InstructionCombining.cpp | 3 +-
.../Instrumentation/BoundsChecking.cpp | 6 +-
.../Instrumentation/MemorySanitizer.cpp | 33 +-
.../Transforms/Scalar/LoopIdiomRecognize.cpp | 52 +-
.../lib/Transforms/Scalar/MemCpyOptimizer.cpp | 48 +-
.../Scalar/RewriteStatepointsForGC.cpp | 7 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 30 +-
llvm/lib/Transforms/Utils/InlineFunction.cpp | 16 +-
.../Utils/ScalarEvolutionExpander.cpp | 6 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 12 +-
.../lib/Transforms/Utils/SimplifyLibCalls.cpp | 50 +-
.../Vectorize/LoadStoreVectorizer.cpp | 8 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 86 ++--
.../Transforms/Vectorize/VectorCombine.cpp | 14 +-
llvm/unittests/Analysis/MemorySSATest.cpp | 8 +-
llvm/unittests/IR/IRBuilderTest.cpp | 44 +-
llvm/unittests/IR/IntrinsicsTest.cpp | 13 +-
llvm/unittests/Transforms/Utils/LocalTest.cpp | 4 +-
61 files changed, 957 insertions(+), 853 deletions(-)
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 99e4b748425c0..09bff9e9b4d00 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -657,27 +657,27 @@ class IRBuilderBase {
///
/// If the pointer isn't an i8*, it will be converted. If alias metadata is
/// specified, it will be added to the instruction.
- CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
- MaybeAlign Align, bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, AAInfo);
}
- LLVM_ABI CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size,
- MaybeAlign Align, bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes());
+ LLVM_ABI Value *CreateMemSet(Value *Ptr, Value *Val, Value *Size,
+ MaybeAlign Align, bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes());
- LLVM_ABI CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
- Value *Val, Value *Size,
- bool IsVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes());
+ LLVM_ABI Value *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
+ Value *Val, Value *Size,
+ bool IsVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes());
/// Create and insert an element unordered-atomic memset of the region of
/// memory starting at the given pointer to the given value.
///
/// If the pointer isn't an i8*, it will be converted. If alias metadata is
/// specified, it will be added to the instruction.
- CallInst *
+ Value *
CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, uint64_t Size,
Align Alignment, uint32_t ElementSize,
const AAMDNodes &AAInfo = AAMDNodes()) {
@@ -704,7 +704,7 @@ class IRBuilderBase {
LLVM_ABI CallInst *CreateFree(Value *Source,
ArrayRef<OperandBundleDef> Bundles = {});
- LLVM_ABI CallInst *
+ LLVM_ABI Value *
CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, Value *Size,
Align Alignment, uint32_t ElementSize,
const AAMDNodes &AAInfo = AAMDNodes());
@@ -714,32 +714,31 @@ class IRBuilderBase {
/// If the pointers aren't i8*, they will be converted. If alias metadata is
/// specified, it will be added to the instruction.
/// and noalias tags.
- CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, uint64_t Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, uint64_t Size,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
isVolatile, AAInfo);
}
- LLVM_ABI CallInst *
- CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign,
- Value *Src, MaybeAlign SrcAlign, Value *Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes());
+ LLVM_ABI Value *CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
+ MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes());
- CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, Value *Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
SrcAlign, Size, isVolatile, AAInfo);
}
- CallInst *CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, Value *Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemTransferInst(Intrinsic::memcpy_inline, Dst, DstAlign, Src,
SrcAlign, Size, isVolatile, AAInfo);
}
@@ -752,22 +751,22 @@ class IRBuilderBase {
///
/// If the pointers aren't i8*, they will be converted. If alias metadata is
/// specified, it will be added to the instruction.
- LLVM_ABI CallInst *CreateElementUnorderedAtomicMemCpy(
+ LLVM_ABI Value *CreateElementUnorderedAtomicMemCpy(
Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
uint32_t ElementSize, const AAMDNodes &AAInfo = AAMDNodes());
- CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, uint64_t Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, uint64_t Size,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
isVolatile, AAInfo);
}
- CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, Value *Size,
- bool isVolatile = false,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ Value *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size,
+ bool isVolatile = false,
+ const AAMDNodes &AAInfo = AAMDNodes()) {
return CreateMemTransferInst(Intrinsic::memmove, Dst, DstAlign, Src,
SrcAlign, Size, isVolatile, AAInfo);
}
@@ -780,113 +779,110 @@ class IRBuilderBase {
///
/// If the pointers aren't i8*, they will be converted. If alias metadata is
/// specified, it will be added to the instruction.
- LLVM_ABI CallInst *CreateElementUnorderedAtomicMemMove(
+ LLVM_ABI Value *CreateElementUnorderedAtomicMemMove(
Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
uint32_t ElementSize, const AAMDNodes &AAInfo = AAMDNodes());
private:
- CallInst *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
+ Value *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
public:
/// Create a sequential vector fadd reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value. An unordered reduction
/// can be created by adding the reassoc fast-math flag to the resulting
/// sequential reduction.
- LLVM_ABI CallInst *CreateFAddReduce(Value *Acc, Value *Src);
+ LLVM_ABI Value *CreateFAddReduce(Value *Acc, Value *Src);
/// Create a sequential vector fmul reduction intrinsic of the source vector.
/// The first parameter is a scalar accumulator value. An unordered reduction
/// can be created by adding the reassoc fast-math flag to the resulting
/// sequential reduction.
- LLVM_ABI CallInst *CreateFMulReduce(Value *Acc, Value *Src);
+ LLVM_ABI Value *CreateFMulReduce(Value *Acc, Value *Src);
/// Create a vector int add reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateAddReduce(Value *Src);
+ LLVM_ABI Value *CreateAddReduce(Value *Src);
/// Create a vector int mul reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateMulReduce(Value *Src);
+ LLVM_ABI Value *CreateMulReduce(Value *Src);
/// Create a vector int AND reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateAndReduce(Value *Src);
+ LLVM_ABI Value *CreateAndReduce(Value *Src);
/// Create a vector int OR reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateOrReduce(Value *Src);
+ LLVM_ABI Value *CreateOrReduce(Value *Src);
/// Create a vector int XOR reduction intrinsic of the source vector.
- LLVM_ABI CallInst *CreateXorReduce(Value *Src);
+ LLVM_ABI Value *CreateXorReduce(Value *Src);
/// Create a vector integer max reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
+ LLVM_ABI Value *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
/// Create a vector integer min reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
+ LLVM_ABI Value *CreateIntMinReduce(Value *Src, bool IsSigned = false);
/// Create a vector float max reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateFPMaxReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMaxReduce(Value *Src);
/// Create a vector float min reduction intrinsic of the source
/// vector.
- LLVM_ABI CallInst *CreateFPMinReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMinReduce(Value *Src);
/// Create a vector float maximum reduction intrinsic of the source
/// vector. This variant follows the NaN and signed zero semantic of
/// llvm.maximum intrinsic.
- LLVM_ABI CallInst *CreateFPMaximumReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMaximumReduce(Value *Src);
/// Create a vector float minimum reduction intrinsic of the source
/// vector. This variant follows the NaN and signed zero semantic of
/// llvm.minimum intrinsic.
- LLVM_ABI CallInst *CreateFPMinimumReduce(Value *Src);
+ LLVM_ABI Value *CreateFPMinimumReduce(Value *Src);
/// Create a lifetime.start intrinsic.
- LLVM_ABI CallInst *CreateLifetimeStart(Value *Ptr);
+ LLVM_ABI Value *CreateLifetimeStart(Value *Ptr);
/// Create a lifetime.end intrinsic.
- LLVM_ABI CallInst *CreateLifetimeEnd(Value *Ptr);
+ LLVM_ABI Value *CreateLifetimeEnd(Value *Ptr);
/// Create a call to invariant.start intrinsic.
///
/// If the pointer isn't i8* it will be converted.
- LLVM_ABI CallInst *CreateInvariantStart(Value *Ptr,
- ConstantInt *Size = nullptr);
+ LLVM_ABI Value *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
/// Create a call to llvm.threadlocal.address intrinsic.
- LLVM_ABI CallInst *CreateThreadLocalAddress(Value *Ptr);
+ LLVM_ABI Value *CreateThreadLocalAddress(Value *Ptr);
/// Create a call to Masked Load intrinsic
- LLVM_ABI CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Store intrinsic
- LLVM_ABI CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
- Value *Mask);
+ LLVM_ABI Value *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+ Value *Mask);
/// Create a call to Masked Gather intrinsic
- LLVM_ABI CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Scatter intrinsic
- LLVM_ABI CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs,
- Align Alignment,
- Value *Mask = nullptr);
+ LLVM_ABI Value *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
+ Value *Mask = nullptr);
/// Create a call to Masked Expand Load intrinsic
- LLVM_ABI CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
- MaybeAlign Align,
- Value *Mask = nullptr,
- Value *PassThru = nullptr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
+ Value *Mask = nullptr,
+ Value *PassThru = nullptr,
+ const Twine &Name = "");
/// Create a call to Masked Compress Store intrinsic
- LLVM_ABI CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align,
- Value *Mask = nullptr);
+ LLVM_ABI Value *CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align,
+ Value *Mask = nullptr);
/// Return an all true boolean vector (mask) with \p NumElts lanes.
Value *getAllOnesMask(ElementCount NumElts) {
@@ -896,29 +892,29 @@ class IRBuilderBase {
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided condition will be true.
- LLVM_ABI CallInst *CreateAssumption(Value *Cond);
+ LLVM_ABI Value *CreateAssumption(Value *Cond);
/// Create an assume intrinsic call that allows the optimizer to
/// assume that the provided operand bundles hold.
- LLVM_ABI CallInst *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
+ LLVM_ABI Value *CreateAssumption(ArrayRef<OperandBundleDef> OpBundles);
/// Create a llvm.experimental.noalias.scope.decl intrinsic call.
- LLVM_ABI Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
- Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
+ LLVM_ABI Value *CreateNoAliasScopeDeclaration(Value *Scope);
+ Value *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
return CreateNoAliasScopeDeclaration(
MetadataAsValue::get(Context, ScopeTag));
}
/// Create a call to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
- LLVM_ABI CallInst *CreateGCStatepointCall(
+ LLVM_ABI Value *CreateGCStatepointCall(
uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
ArrayRef<Value *> GCArgs, const Twine &Name = "");
/// Create a call to the experimental.gc.statepoint intrinsic to
/// start a new statepoint sequence.
- LLVM_ABI CallInst *
+ LLVM_ABI Value *
CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
FunctionCallee ActualCallee, uint32_t Flags,
ArrayRef<Value *> CallArgs,
@@ -929,7 +925,7 @@ class IRBuilderBase {
/// Conveninence function for the common case when CallArgs are filled
/// in using ArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
/// .get()'ed to get the Value pointer.
- LLVM_ABI CallInst *
+ LLVM_ABI Value *
CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
FunctionCallee ActualCallee, ArrayRef<Use> CallArgs,
std::optional<ArrayRef<Value *>> DeoptArgs,
@@ -965,24 +961,24 @@ class IRBuilderBase {
/// Create a call to the experimental.gc.result intrinsic to extract
/// the result from a call wrapped in a statepoint.
- LLVM_ABI CallInst *CreateGCResult(Instruction *Statepoint, Type *ResultType,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateGCResult(Instruction *Statepoint, Type *ResultType,
+ const Twine &Name = "");
/// Create a call to the experimental.gc.relocate intrinsics to
/// project the relocated value of one pointer from the statepoint.
- LLVM_ABI CallInst *CreateGCRelocate(Instruction *Statepoint, int BaseOffset,
- int DerivedOffset, Type *ResultType,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateGCRelocate(Instruction *Statepoint, int BaseOffset,
+ int DerivedOffset, Type *ResultType,
+ const Twine &Name = "");
/// Create a call to the experimental.gc.pointer.base intrinsic to get the
/// base pointer for the specified derived pointer.
- LLVM_ABI CallInst *CreateGCGetPointerBase(Value *DerivedPtr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateGCGetPointerBase(Value *DerivedPtr,
+ const Twine &Name = "");
/// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
/// the offset of the specified derived pointer from its base.
- LLVM_ABI CallInst *CreateGCGetPointerOffset(Value *DerivedPtr,
- const Twine &Name = "");
+ LLVM_ABI Value *CreateGCGetPointerOffset(Value *DerivedPtr,
+ const Twine &Name = "");
/// Create a call to llvm.vscale.<Ty>().
Value *CreateVScale(Type *Ty, const Twine &Name = "") {
@@ -1021,26 +1017,26 @@ class IRBuilderBase {
/// Create a call to intrinsic \p ID with \p Args, mangled using
/// \p OverloadTypes. If \p FMFSource is provided, copy fast-math-flags from
/// that instruction to the intrinsic.
- LLVM_ABI CallInst *CreateIntrinsic(Intrinsic::ID ID,
- ArrayRef<Type *> OverloadTypes,
- ArrayRef<Value *> Args,
- FMFSource FMFSource = {},
- const Twine &Name = "",
- ArrayRef<OperandBundleDef> OpBundles = {});
+ LLVM_ABI Value *CreateIntrinsic(Intrinsic::ID ID,
+ ArrayRef<Type *> OverloadTypes,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource = {},
+ const Twine &Name = "",
+ ArrayRef<OperandBundleDef> OpBundles = {});
/// Create a call to intrinsic \p ID with \p RetTy and \p Args. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
/// the intrinsic.
- LLVM_ABI CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
- ArrayRef<Value *> Args,
- FMFSource FMFSource = {},
- const Twine &Name = "");
+ LLVM_ABI Value *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource = {},
+ const Twine &Name = "");
/// Create a call to non-overloaded intrinsic \p ID with \p Args. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
/// the intrinsic.
- CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Args,
- FMFSource FMFSource = {}, const Twine &Name = "") {
+ Value *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Args,
+ FMFSource FMFSource = {}, const Twine &Name = "") {
return CreateIntrinsic(ID, /*Types=*/{}, Args, FMFSource, Name);
}
@@ -1125,49 +1121,49 @@ class IRBuilderBase {
}
/// Create a call to the arithmetic_fence intrinsic.
- CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
- const Twine &Name = "") {
+ Value *CreateArithmeticFence(Value *Val, Type *DstType,
+ const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
- const Twine &Name = "") {
+ Value *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
+ const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::vector_extract,
{DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, uint64_t Idx,
- const Twine &Name = "") {
+ Value *CreateExtractVector(Type *DstType, Value *SrcVec, uint64_t Idx,
+ const Twine &Name = "") {
return CreateExtractVector(DstType, SrcVec, getInt64(Idx), Name);
}
/// Create a call to the vector.insert intrinsic.
- CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
- Value *Idx, const Twine &Name = "") {
+ Value *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
+ Value *Idx, const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::vector_insert,
{DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
nullptr, Name);
}
/// Create a call to the vector.extract intrinsic.
- CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
- uint64_t Idx, const Twine &Name = "") {
+ Value *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
+ uint64_t Idx, const Twine &Name = "") {
return CreateInsertVector(DstType, SrcVec, SubVec, getInt64(Idx), Name);
}
/// Create a call to llvm.stacksave
- CallInst *CreateStackSave(const Twine &Name = "") {
+ Value *CreateStackSave(const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
return CreateIntrinsic(Intrinsic::stacksave, {DL.getAllocaPtrType(Context)},
{}, nullptr, Name);
}
/// Create a call to llvm.stackrestore
- CallInst *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
+ Value *CreateStackRestore(Value *Ptr, const Twine &Name = "") {
return CreateIntrinsic(Intrinsic::stackrestore, {Ptr->getType()}, {Ptr},
nullptr, Name);
}
@@ -1183,9 +1179,9 @@ class IRBuilderBase {
private:
/// Create a call to a masked intrinsic with given Id.
- CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name = "");
+ Value *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name = "");
//===--------------------------------------------------------------------===//
// Instruction creation methods: Terminators
@@ -1832,19 +1828,19 @@ class IRBuilderBase {
/// the created intrinsic call according to \p Rounding and \p
/// Except and it sets \p FPMathTag as the 'fpmath' metadata, using
/// defaults if a value equals nullopt/null.
- LLVM_ABI CallInst *CreateConstrainedFPIntrinsic(
+ LLVM_ABI Value *CreateConstrainedFPIntrinsic(
Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
- LLVM_ABI CallInst *CreateConstrainedFPBinOp(
+ LLVM_ABI Value *CreateConstrainedFPBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
- LLVM_ABI CallInst *CreateConstrainedFPUnroundedBinOp(
+ LLVM_ABI Value *CreateConstrainedFPUnroundedBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -1913,13 +1909,14 @@ class IRBuilderBase {
return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
}
- CallInst *CreateStructuredAlloca(Type *BaseType, const Twine &Name = "") {
+ Value *CreateStructuredAlloca(Type *BaseType, const Twine &Name = "") {
const DataLayout &DL = BB->getDataLayout();
PointerType *PtrTy = DL.getAllocaPtrType(Context);
- CallInst *Output =
+ Value *Output =
CreateIntrinsic(Intrinsic::structured_alloca, {PtrTy}, {}, {}, Name);
- Output->addRetAttr(
- Attribute::get(getContext(), Attribute::ElementType, BaseType));
+ if (auto *CI = dyn_cast<CallInst>(Output))
+ CI->addRetAttr(
+ Attribute::get(getContext(), Attribute::ElementType, BaseType));
return Output;
}
@@ -2003,17 +2000,18 @@ class IRBuilderBase {
new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID, Elementwise));
}
- CallInst *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
- ArrayRef<Value *> Indices,
- const Twine &Name = "") {
+ Value *CreateStructuredGEP(Type *BaseType, Value *PtrBase,
+ ArrayRef<Value *> Indices,
+ const Twine &Name = "") {
SmallVector<Value *> Args;
Args.push_back(PtrBase);
llvm::append_range(Args, Indices);
- CallInst *Output = CreateIntrinsic(Intrinsic::structured_gep,
- {PtrBase->getType()}, Args, {}, Name);
- Output->addParamAttr(
- 0, Attribute::get(getContext(), Attribute::ElementType, BaseType));
+ Value *Output = CreateIntrinsic(Intrinsic::structured_gep,
+ {PtrBase->getType()}, Args, {}, Name);
+ if (auto *CI = dyn_cast<CallInst>(Output))
+ CI->addParamAttr(
+ 0, Attribute::get(getContext(), Attribute::ElementType, BaseType));
return Output;
}
@@ -2352,7 +2350,7 @@ class IRBuilderBase {
return CreateCast(CastOp, V, DestTy, Name, FPMathTag);
}
- LLVM_ABI CallInst *CreateConstrainedFPCast(
+ LLVM_ABI Value *CreateConstrainedFPCast(
Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource = {},
const Twine &Name = "", MDNode *FPMathTag = nullptr,
std::optional<RoundingMode> Rounding = std::nullopt,
@@ -2537,7 +2535,7 @@ class IRBuilderBase {
FMFSource FMFSource, bool IsSignaling);
public:
- LLVM_ABI CallInst *CreateConstrainedFPCmp(
+ LLVM_ABI Value *CreateConstrainedFPCmp(
Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
const Twine &Name = "",
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -2595,7 +2593,7 @@ class IRBuilderBase {
OpBundles, Name, FPMathTag);
}
- LLVM_ABI CallInst *CreateConstrainedFPCall(
+ LLVM_ABI Value *CreateConstrainedFPCall(
Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
std::optional<RoundingMode> Rounding = std::nullopt,
std::optional<fp::ExceptionBehavior> Except = std::nullopt);
@@ -2809,9 +2807,8 @@ class IRBuilderBase {
/// Helper function that creates an assume intrinsic call that
/// represents an alignment assumption on the provided pointer \p PtrValue
/// with offset \p OffsetValue and alignment value \p AlignValue.
- CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
- Value *PtrValue, Value *AlignValue,
- Value *OffsetValue);
+ Value *CreateAlignmentAssumptionHelper(const DataLayout &DL, Value *PtrValue,
+ Value *AlignValue, Value *OffsetValue);
public:
/// Create an assume intrinsic call that represents an alignment
@@ -2820,10 +2817,9 @@ class IRBuilderBase {
/// An optional offset can be provided, and if it is provided, the offset
/// must be subtracted from the provided pointer to get the pointer with the
/// specified alignment.
- LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- uint64_t Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue, uint64_t Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents an alignment
/// assumption on the provided pointer.
@@ -2834,19 +2830,18 @@ class IRBuilderBase {
///
/// This overload handles the condition where the Alignment is dependent
/// on an existing value rather than a static value.
- LLVM_ABI CallInst *CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- Value *Alignment,
- Value *OffsetValue = nullptr);
+ LLVM_ABI Value *CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue, Value *Alignment,
+ Value *OffsetValue = nullptr);
/// Create an assume intrinsic call that represents a dereferencable
/// assumption on the provided pointer.
- LLVM_ABI CallInst *CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue);
+ LLVM_ABI Value *CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue);
/// Create an assume intrinsic call that represents a nonnull assumption on
/// the provided pointer.
- LLVM_ABI CallInst *CreateNonnullAssumption(Value *PtrValue);
+ LLVM_ABI Value *CreateNonnullAssumption(Value *PtrValue);
};
/// This provides a uniform API for creating instructions and inserting
diff --git a/llvm/lib/CodeGen/InlineAsmPrepare.cpp b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
index 0bbf55c3d31e4..8db216c26f858 100644
--- a/llvm/lib/CodeGen/InlineAsmPrepare.cpp
+++ b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
@@ -188,8 +188,8 @@ static bool insertIntrinsicCalls(CallBrInst *CBR, DominatorTree &DT) {
continue;
Builder.SetInsertPoint(&*IndDest->begin());
- CallInst *Intrinsic = Builder.CreateIntrinsic(
- CBR->getType(), Intrinsic::callbr_landingpad, {CBR});
+ CallInst *Intrinsic = cast<CallInst>(Builder.CreateIntrinsic(
+ CBR->getType(), Intrinsic::callbr_landingpad, {CBR}));
SSAUpdate.AddAvailableValue(IndDest, Intrinsic);
updateSSA(DT, CBR, Intrinsic, SSAUpdate);
Changed = true;
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index 152dcca209298..2a4c143de34ca 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -556,9 +556,10 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
if (FrameAlignment > StackAlignment) {
// Re-align the base pointer according to the max requested alignment.
IRB.SetInsertPoint(BasePointer->getNextNode());
- BasePointer = IRB.CreateIntrinsic(
+ BasePointer = cast<Instruction>(IRB.CreateIntrinsic(
StackPtrTy, Intrinsic::ptrmask,
- {BasePointer, ConstantInt::get(AddrTy, ~(FrameAlignment.value() - 1))});
+ {BasePointer,
+ ConstantInt::get(AddrTy, ~(FrameAlignment.value() - 1))}));
}
IRB.SetInsertPoint(BasePointer->getNextNode());
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 0770f0f0ff060..1c4cad6408617 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5127,7 +5127,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
return;
};
- CallInst *NewCall = nullptr;
+ Value *NewCall = nullptr;
switch (NewFn->getIntrinsicID()) {
default: {
DefaultCase();
@@ -5285,8 +5285,9 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
Value *Dynamic =
CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
- NewCall = Builder.CreateCall(
- NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
+ NewCall =
+ Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
+ NullIsUnknownSize, Dynamic});
break;
}
@@ -5625,8 +5626,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
{OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
- NewCall->setAttributes(NewAttrs);
auto *MemCI = cast<MemIntrinsic>(NewCall);
+ MemCI->setAttributes(NewAttrs);
// All mem intrinsics support dest alignment.
const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
MemCI->setDestAlignment(Align->getMaybeAlignValue());
@@ -5693,8 +5694,10 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
llvm_unreachable("Unexpected intrinsic ID");
}
// Previous metadata is still valid.
- NewCall->copyMetadata(*CI);
- NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ if (auto *NewCI = dyn_cast<CallInst>(NewCall)) {
+ NewCI->copyMetadata(*CI);
+ NewCI->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+ }
break;
}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 876d642b98fb0..3e7526d689c68 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -248,73 +248,79 @@ Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
return ConstantVector::get(Indices);
}
-CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
- MaybeAlign Align, bool isVolatile,
- const AAMDNodes &AAInfo) {
+Value *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
+ MaybeAlign Align, bool isVolatile,
+ const AAMDNodes &AAInfo) {
Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
Type *Tys[] = {Ptr->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(Intrinsic::memset, Tys, Ops);
+ Value *V = CreateIntrinsic(Intrinsic::memset, Tys, Ops);
- if (Align)
- cast<MemSetInst>(CI)->setDestAlignment(*Align);
- CI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *CI = dyn_cast<MemSetInst>(V)) {
+ if (Align)
+ cast<MemSetInst>(CI)->setDestAlignment(*Align);
+ CI->setAAMetadata(AAInfo);
+ }
+ return V;
}
-CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
- Value *Val, Value *Size,
- bool IsVolatile,
- const AAMDNodes &AAInfo) {
+Value *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
+ Value *Val, Value *Size,
+ bool IsVolatile,
+ const AAMDNodes &AAInfo) {
Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
Type *Tys[] = {Dst->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(Intrinsic::memset_inline, Tys, Ops);
+ Value *V = CreateIntrinsic(Intrinsic::memset_inline, Tys, Ops);
- if (DstAlign)
- cast<MemSetInst>(CI)->setDestAlignment(*DstAlign);
- CI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *CI = dyn_cast<MemSetInst>(V)) {
+ if (DstAlign)
+ CI->setDestAlignment(*DstAlign);
+ CI->setAAMetadata(AAInfo);
+ }
+ return V;
}
-CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
+Value *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
const AAMDNodes &AAInfo) {
Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
Type *Tys[] = {Ptr->getType(), Size->getType()};
- CallInst *CI =
+ Value *V =
CreateIntrinsic(Intrinsic::memset_element_unordered_atomic, Tys, Ops);
-
- cast<AnyMemSetInst>(CI)->setDestAlignment(Alignment);
- CI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *CI = dyn_cast<AnyMemSetInst>(V)) {
+ CI->setDestAlignment(Alignment);
+ CI->setAAMetadata(AAInfo);
+ }
+ return V;
}
-CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
- MaybeAlign DstAlign, Value *Src,
- MaybeAlign SrcAlign, Value *Size,
- bool isVolatile,
- const AAMDNodes &AAInfo) {
+Value *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
+ MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size,
+ bool isVolatile,
+ const AAMDNodes &AAInfo) {
assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
IntrID == Intrinsic::memmove) &&
"Unexpected intrinsic ID");
Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI = CreateIntrinsic(IntrID, Tys, Ops);
+ Value *V = CreateIntrinsic(IntrID, Tys, Ops);
- auto* MCI = cast<MemTransferInst>(CI);
- if (DstAlign)
- MCI->setDestAlignment(*DstAlign);
- if (SrcAlign)
- MCI->setSourceAlignment(*SrcAlign);
- MCI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *MCI = dyn_cast<MemTransferInst>(V)) {
+ if (DstAlign)
+ MCI->setDestAlignment(*DstAlign);
+ if (SrcAlign)
+ MCI->setSourceAlignment(*SrcAlign);
+ MCI->setAAMetadata(AAInfo);
+ }
+ return V;
}
-CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
+Value *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
uint32_t ElementSize, const AAMDNodes &AAInfo) {
assert(DstAlign >= ElementSize &&
@@ -324,15 +330,16 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI =
+ Value *V =
CreateIntrinsic(Intrinsic::memcpy_element_unordered_atomic, Tys, Ops);
// Set the alignment of the pointer args.
- auto *AMCI = cast<AnyMemCpyInst>(CI);
- AMCI->setDestAlignment(DstAlign);
- AMCI->setSourceAlignment(SrcAlign);
- AMCI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *AMCI = dyn_cast<AnyMemCpyInst>(V)) {
+ AMCI->setDestAlignment(DstAlign);
+ AMCI->setSourceAlignment(SrcAlign);
+ AMCI->setAAMetadata(AAInfo);
+ }
+ return V;
}
/// isConstantOne - Return true only if val is constant int 1
@@ -413,7 +420,7 @@ CallInst *IRBuilderBase::CreateFree(Value *Source,
return Result;
}
-CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
+Value *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
uint32_t ElementSize, const AAMDNodes &AAInfo) {
assert(DstAlign >= ElementSize &&
@@ -423,93 +430,97 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
- CallInst *CI =
+ Value *V =
CreateIntrinsic(Intrinsic::memmove_element_unordered_atomic, Tys, Ops);
- // Set the alignment of the pointer args.
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
- CI->setAAMetadata(AAInfo);
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ // Set the alignment of the pointer args.
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), DstAlign));
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), SrcAlign));
+ CI->setAAMetadata(AAInfo);
+ }
+ return V;
}
-CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
+Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
Value *Ops[] = {Src};
Type *Tys[] = { Src->getType() };
return CreateIntrinsic(ID, Tys, Ops);
}
-CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
+Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
Value *Ops[] = {Acc, Src};
return CreateIntrinsic(Intrinsic::vector_reduce_fadd, {Src->getType()}, Ops);
}
-CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
+Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
Value *Ops[] = {Acc, Src};
return CreateIntrinsic(Intrinsic::vector_reduce_fmul, {Src->getType()}, Ops);
}
-CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
+Value *IRBuilderBase::CreateAddReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
}
-CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
+Value *IRBuilderBase::CreateMulReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
}
-CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
+Value *IRBuilderBase::CreateAndReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
}
-CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
+Value *IRBuilderBase::CreateOrReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
}
-CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
+Value *IRBuilderBase::CreateXorReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
}
-CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
+Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
auto ID =
IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
return getReductionIntrinsic(ID, Src);
}
-CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
+Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
auto ID =
IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
return getReductionIntrinsic(ID, Src);
}
-CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
}
-CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
}
-CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);
}
-CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
+Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
}
-CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
+Value *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.start only applies to pointers.");
return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr});
}
-CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
+Value *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
assert(isa<PointerType>(Ptr->getType()) &&
"lifetime.end only applies to pointers.");
return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr});
}
-CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
+Value *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
assert(isa<PointerType>(Ptr->getType()) &&
"invariant.start only applies to pointers.");
@@ -533,32 +544,32 @@ static MaybeAlign getAlign(Value *Ptr) {
return {};
}
-CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
+Value *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
"threadlocal_address only applies to thread local variables.");
- CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
- {Ptr->getType()}, {Ptr});
- if (MaybeAlign A = getAlign(Ptr)) {
+ Value *V = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
+ {Ptr->getType()}, {Ptr});
+ auto *CI = dyn_cast<CallInst>(V);
+ if (MaybeAlign A = getAlign(Ptr); CI && A) {
CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));
CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));
}
- return CI;
+ return V;
}
-CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
+Value *IRBuilderBase::CreateAssumption(Value *Cond) {
assert(Cond->getType() == getInt1Ty() &&
"an assumption condition must be of type i1");
return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, {Cond});
}
-CallInst *
-IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
+Value *IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
Value *Args[] = {ConstantInt::getTrue(getContext())};
return CreateIntrinsic(Intrinsic::assume, /*OverloadTypes=*/{}, Args,
/*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
}
-Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
+Value *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
return CreateIntrinsic(Intrinsic::experimental_noalias_scope_decl, {},
{Scope});
}
@@ -572,9 +583,9 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
- Value *Mask, Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
+ Value *Mask, Value *PassThru,
+ const Twine &Name) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
@@ -582,10 +593,12 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
PassThru = PoisonValue::get(Ty);
Type *OverloadedTypes[] = { Ty, PtrTy };
Value *Ops[] = {Ptr, Mask, PassThru};
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name);
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked Store intrinsic.
@@ -594,27 +607,29 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
/// \p Alignment - alignment of the destination location
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
- Align Alignment, Value *Mask) {
+Value *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
+ Value *Mask) {
auto *PtrTy = cast<PointerType>(Ptr->getType());
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *OverloadedTypes[] = { DataTy, PtrTy };
Value *Ops[] = {Val, Ptr, Mask};
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked intrinsic, with given intrinsic Id,
/// an array of operands - Ops, and an array of overloaded types -
/// OverloadedTypes.
-CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
- ArrayRef<Value *> Ops,
- ArrayRef<Type *> OverloadedTypes,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
+ ArrayRef<Value *> Ops,
+ ArrayRef<Type *> OverloadedTypes,
+ const Twine &Name) {
return CreateIntrinsic(Id, OverloadedTypes, Ops, {}, Name);
}
@@ -627,10 +642,9 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
- Align Alignment, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
+ Value *Mask, Value *PassThru,
+ const Twine &Name) {
auto *VecTy = cast<VectorType>(Ty);
ElementCount NumElts = VecTy->getElementCount();
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
@@ -647,10 +661,12 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
- OverloadedTypes, Name);
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
+ OverloadedTypes, Name);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to a Masked Scatter intrinsic.
@@ -660,8 +676,8 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
/// \p Align - alignment for one element
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
- Align Alignment, Value *Mask) {
+Value *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
+ Align Alignment, Value *Mask) {
auto *PtrsTy = cast<VectorType>(Ptrs->getType());
auto *DataTy = cast<VectorType>(Data->getType());
ElementCount NumElts = PtrsTy->getElementCount();
@@ -674,10 +690,12 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
// We specify only one type when we create this intrinsic. Types of other
// arguments are derived from this type.
- CallInst *CI =
+ Value *V =
CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), Alignment));
+ return V;
}
/// Create a call to Masked Expand Load intrinsic
@@ -689,10 +707,10 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
/// of the result
/// \p Name - name of the result variable
-CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
- MaybeAlign Align, Value *Mask,
- Value *PassThru,
- const Twine &Name) {
+Value *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
+ MaybeAlign Align, Value *Mask,
+ Value *PassThru,
+ const Twine &Name) {
assert(Ty->isVectorTy() && "Type should be vector");
assert(Mask && "Mask should not be all-ones (null)");
if (!PassThru)
@@ -700,11 +718,13 @@ CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {Ty, PtrTy};
Value *Ops[] = {Ptr, Mask, PassThru};
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
- OverloadedTypes, Name);
- if (Align)
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *Align));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
+ OverloadedTypes, Name);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (Align)
+ CI->addParamAttr(0,
+ Attribute::getWithAlignment(CI->getContext(), *Align));
+ return V;
}
/// Create a call to Masked Compress Store intrinsic
@@ -713,20 +733,21 @@ CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
/// \p Align - alignment of \p Ptr
/// \p Mask - vector of booleans which indicates what vector lanes should
/// be accessed in memory
-CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
- MaybeAlign Align,
- Value *Mask) {
+Value *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
+ MaybeAlign Align, Value *Mask) {
Type *DataTy = Val->getType();
assert(DataTy->isVectorTy() && "Val should be a vector");
assert(Mask && "Mask should not be all-ones (null)");
Type *PtrTy = Ptr->getType();
Type *OverloadedTypes[] = {DataTy, PtrTy};
Value *Ops[] = {Val, Ptr, Mask};
- CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
- OverloadedTypes);
- if (Align)
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), *Align));
- return CI;
+ Value *V = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
+ OverloadedTypes);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (Align)
+ CI->addParamAttr(1,
+ Attribute::getWithAlignment(CI->getContext(), *Align));
+ return V;
}
template <typename T0>
@@ -765,7 +786,7 @@ getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
}
template <typename T0, typename T1, typename T2, typename T3>
-static CallInst *CreateGCStatepointCallCommon(
+static Value *CreateGCStatepointCallCommon(
IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
std::optional<ArrayRef<T1>> TransitionArgs,
@@ -780,16 +801,17 @@ static CallInst *CreateGCStatepointCallCommon(
std::vector<Value *> Args = getStatepointArgs(
*Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
- CallInst *CI = Builder->CreateCall(
+ Value *V = Builder->CreateCall(
FnStatepoint, Args,
getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
- CI->addParamAttr(2,
- Attribute::get(Builder->getContext(), Attribute::ElementType,
- ActualCallee.getFunctionType()));
- return CI;
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->addParamAttr(2, Attribute::get(Builder->getContext(),
+ Attribute::ElementType,
+ ActualCallee.getFunctionType()));
+ return V;
}
-CallInst *IRBuilderBase::CreateGCStatepointCall(
+Value *IRBuilderBase::CreateGCStatepointCall(
uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
ArrayRef<Value *> GCArgs, const Twine &Name) {
@@ -798,7 +820,7 @@ CallInst *IRBuilderBase::CreateGCStatepointCall(
CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
}
-CallInst *IRBuilderBase::CreateGCStatepointCall(
+Value *IRBuilderBase::CreateGCStatepointCall(
uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
uint32_t Flags, ArrayRef<Value *> CallArgs,
std::optional<ArrayRef<Use>> TransitionArgs,
@@ -809,7 +831,7 @@ CallInst *IRBuilderBase::CreateGCStatepointCall(
DeoptArgs, GCArgs, Name);
}
-CallInst *IRBuilderBase::CreateGCStatepointCall(
+Value *IRBuilderBase::CreateGCStatepointCall(
uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
ArrayRef<Value *> GCArgs, const Twine &Name) {
@@ -878,8 +900,8 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
GCArgs, Name);
}
-CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
- Type *ResultType, const Twine &Name) {
+Value *IRBuilderBase::CreateGCResult(Instruction *Statepoint, Type *ResultType,
+ const Twine &Name) {
Intrinsic::ID ID = Intrinsic::experimental_gc_result;
Type *Types[] = {ResultType};
@@ -887,9 +909,9 @@ CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
return CreateIntrinsic(ID, Types, Args, {}, Name);
}
-CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
- int BaseOffset, int DerivedOffset,
- Type *ResultType, const Twine &Name) {
+Value *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint, int BaseOffset,
+ int DerivedOffset, Type *ResultType,
+ const Twine &Name) {
Type *Types[] = {ResultType};
Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
@@ -897,15 +919,15 @@ CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
Name);
}
-CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
- const Twine &Name) {
+Value *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
+ const Twine &Name) {
Type *PtrTy = DerivedPtr->getType();
return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_base,
{PtrTy, PtrTy}, {DerivedPtr}, {}, Name);
}
-CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
- const Twine &Name) {
+Value *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
+ const Twine &Name) {
Type *PtrTy = DerivedPtr->getType();
return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_offset, {PtrTy},
{DerivedPtr}, {}, Name);
@@ -933,27 +955,26 @@ Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
}
-CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
- ArrayRef<Type *> OverloadTypes,
- ArrayRef<Value *> Args,
- FMFSource FMFSource, const Twine &Name,
- ArrayRef<OperandBundleDef> OpBundles) {
+Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
+ ArrayRef<Type *> OverloadTypes,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource, const Twine &Name,
+ ArrayRef<OperandBundleDef> OpBundles) {
Module *M = BB->getModule();
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTypes);
return createCallHelper(Fn, Args, Name, FMFSource, OpBundles);
}
-CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
- ArrayRef<Value *> Args,
- FMFSource FMFSource,
- const Twine &Name) {
+Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
+ ArrayRef<Value *> Args,
+ FMFSource FMFSource, const Twine &Name) {
Module *M = BB->getModule();
SmallVector<Type *> ArgTys = llvm::map_to_vector(Args, &Value::getType);
Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, RetTy, ArgTys);
return createCallHelper(Fn, Args, Name, FMFSource);
}
-CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
+Value *IRBuilderBase::CreateConstrainedFPBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
std::optional<fp::ExceptionBehavior> Except) {
@@ -962,14 +983,16 @@ CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C = CreateIntrinsic(ID, {L->getType()},
- {L, R, RoundingV, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ Value *C = CreateIntrinsic(ID, {L->getType()}, {L, R, RoundingV, ExceptV},
+ nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
-CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
+Value *IRBuilderBase::CreateConstrainedFPIntrinsic(
Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
std::optional<RoundingMode> Rounding,
@@ -983,13 +1006,15 @@ CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
ExtArgs.push_back(RoundingV);
ExtArgs.push_back(ExceptV);
- CallInst *C = CreateIntrinsic(ID, Types, ExtArgs, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ Value *C = CreateIntrinsic(ID, Types, ExtArgs, nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
-CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
+Value *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag,
std::optional<fp::ExceptionBehavior> Except) {
@@ -997,10 +1022,12 @@ CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C =
+ Value *C =
CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
- setFPAttrs(C, FPMathTag, UseFMF);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
@@ -1019,7 +1046,7 @@ Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
llvm_unreachable("Unexpected opcode!");
}
-CallInst *IRBuilderBase::CreateConstrainedFPCast(
+Value *IRBuilderBase::CreateConstrainedFPCast(
Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
std::optional<fp::ExceptionBehavior> Except) {
@@ -1027,7 +1054,7 @@ CallInst *IRBuilderBase::CreateConstrainedFPCast(
FastMathFlags UseFMF = FMFSource.get(FMF);
- CallInst *C;
+ Value *C;
if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {
Value *RoundingV = getConstrainedFPRounding(Rounding);
C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
@@ -1036,10 +1063,12 @@ CallInst *IRBuilderBase::CreateConstrainedFPCast(
C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
Name);
- setConstrainedFPCallAttr(C);
+ if (auto *CI = dyn_cast<CallInst>(C)) {
+ setConstrainedFPCallAttr(CI);
- if (isa<FPMathOperator>(C))
- setFPAttrs(C, FPMathTag, UseFMF);
+ if (isa<FPMathOperator>(CI))
+ setFPAttrs(CI, FPMathTag, UseFMF);
+ }
return C;
}
@@ -1060,19 +1089,20 @@ Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
Name);
}
-CallInst *IRBuilderBase::CreateConstrainedFPCmp(
+Value *IRBuilderBase::CreateConstrainedFPCmp(
Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
Value *PredicateV = getConstrainedFPPredicate(P);
Value *ExceptV = getConstrainedFPExcept(Except);
- CallInst *C = CreateIntrinsic(ID, {L->getType()},
- {L, R, PredicateV, ExceptV}, nullptr, Name);
- setConstrainedFPCallAttr(C);
+ Value *C = CreateIntrinsic(ID, {L->getType()}, {L, R, PredicateV, ExceptV},
+ nullptr, Name);
+ if (auto *CI = dyn_cast<CallInst>(C))
+ setConstrainedFPCallAttr(CI);
return C;
}
-CallInst *IRBuilderBase::CreateConstrainedFPCall(
+Value *IRBuilderBase::CreateConstrainedFPCall(
Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
std::optional<RoundingMode> Rounding,
std::optional<fp::ExceptionBehavior> Except) {
@@ -1082,8 +1112,9 @@ CallInst *IRBuilderBase::CreateConstrainedFPCall(
UseArgs.push_back(getConstrainedFPRounding(Rounding));
UseArgs.push_back(getConstrainedFPExcept(Except));
- CallInst *C = CreateCall(Callee, UseArgs, Name);
- setConstrainedFPCallAttr(C);
+ Value *C = CreateCall(Callee, UseArgs, Name);
+ if (auto *CI = dyn_cast<CallInst>(C))
+ setConstrainedFPCallAttr(CI);
return C;
}
@@ -1300,15 +1331,16 @@ Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
Value *DimV = getInt32(Dimension);
- CallInst *Fn =
- CreateIntrinsic(Intrinsic::preserve_array_access_index,
- {ResultType, BaseType}, {Base, DimV, LastIndexV});
- Fn->addParamAttr(
- 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ Value *V = CreateIntrinsic(Intrinsic::preserve_array_access_index,
+ {ResultType, BaseType}, {Base, DimV, LastIndexV});
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ CI->addParamAttr(
+ 0, Attribute::get(CI->getContext(), Attribute::ElementType, ElTy));
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ }
- return Fn;
+ return V;
}
Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
@@ -1318,12 +1350,13 @@ Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
auto *BaseType = Base->getType();
Value *DIIndex = getInt32(FieldIndex);
- CallInst *Fn = CreateIntrinsic(Intrinsic::preserve_union_access_index,
- {BaseType, BaseType}, {Base, DIIndex});
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ Value *V = CreateIntrinsic(Intrinsic::preserve_union_access_index,
+ {BaseType, BaseType}, {Base, DIIndex});
+ if (auto *CI = dyn_cast<CallInst>(V))
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
- return Fn;
+ return V;
}
Value *IRBuilderBase::CreatePreserveStructAccessIndex(
@@ -1339,15 +1372,15 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex(
GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
Value *DIIndex = getInt32(FieldIndex);
- CallInst *Fn =
- CreateIntrinsic(Intrinsic::preserve_struct_access_index,
- {ResultType, BaseType}, {Base, GEPIndex, DIIndex});
- Fn->addParamAttr(
- 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
- if (DbgInfo)
- Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
-
- return Fn;
+ Value *V = CreateIntrinsic(Intrinsic::preserve_struct_access_index,
+ {ResultType, BaseType}, {Base, GEPIndex, DIIndex});
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ CI->addParamAttr(
+ 0, Attribute::get(CI->getContext(), Attribute::ElementType, ElTy));
+ if (DbgInfo)
+ CI->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
+ }
+ return V;
}
Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
@@ -1356,10 +1389,10 @@ Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
{FPNum, TestV});
}
-CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
- Value *PtrValue,
- Value *AlignValue,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
+ Value *PtrValue,
+ Value *AlignValue,
+ Value *OffsetValue) {
SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
if (OffsetValue)
Vals.push_back(OffsetValue);
@@ -1367,10 +1400,10 @@ CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
return CreateAssumption({AlignOpB});
}
-CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- uint64_t Alignment,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ uint64_t Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
assert(Alignment != 0 && "Invalid Alignment");
@@ -1380,17 +1413,17 @@ CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
}
-CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
- Value *PtrValue,
- Value *Alignment,
- Value *OffsetValue) {
+Value *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
+ Value *PtrValue,
+ Value *Alignment,
+ Value *OffsetValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create an alignment assumption on a non-pointer?");
return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
}
-CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
- Value *SizeValue) {
+Value *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
+ Value *SizeValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a deferenceable assumption on a non-pointer?");
SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
@@ -1398,7 +1431,7 @@ CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
return CreateAssumption({DereferenceableOpB});
}
-CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
+Value *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
assert(isa<PointerType>(PtrValue->getType()) &&
"trying to create a nonnull assumption on a non-pointer?");
return CreateAssumption(OperandBundleDef("nonnull", PtrValue));
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bb3d7b560b534..a81f265fa9d10 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -31352,10 +31352,11 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
const DataLayout &DL = M->getDataLayout();
IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
- CallInst *CI = Builder.CreateIntrinsic(Int, Tys, Addr);
- CI->addParamAttr(0, Attribute::get(Builder.getContext(),
- Attribute::ElementType, IntEltTy));
- Value *Trunc = Builder.CreateTrunc(CI, IntEltTy);
+ Value *Intr = Builder.CreateIntrinsic(Int, Tys, Addr);
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(0, Attribute::get(Builder.getContext(),
+ Attribute::ElementType, IntEltTy));
+ Value *Trunc = Builder.CreateTrunc(Intr, IntEltTy);
return Builder.CreateBitCast(Trunc, ValueTy);
}
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 7ef18fa18255d..447a8bfa67ff2 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -477,9 +477,9 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
assert(PrologueBB);
IRBuilder<> IRB(&PrologueBB->front());
- Instruction *Base =
+ Instruction *Base = cast<Instruction>(
IRB.CreateIntrinsic(Intrinsic::aarch64_irg_sp, {},
- {Constant::getNullValue(IRB.getInt64Ty())});
+ {Constant::getNullValue(IRB.getInt64Ty())}));
Base->setName("basetag");
const Triple &TargetTriple = M.getTargetTriple();
// This ABI will make it into Android API level 35.
@@ -583,10 +583,10 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
NextTag = (NextTag + 1) % 16;
// Replace alloca with tagp(alloca).
IRBuilder<> IRB(Info.AI->getNextNode());
- Instruction *TagPCall =
+ Instruction *TagPCall = cast<Instruction>(
IRB.CreateIntrinsic(Intrinsic::aarch64_tagp, {Info.AI->getType()},
{Constant::getNullValue(Info.AI->getType()), Base,
- ConstantInt::get(IRB.getInt64Ty(), Tag)});
+ ConstantInt::get(IRB.getInt64Ty(), Tag)}));
if (Info.AI->hasName())
TagPCall->setName(Info.AI->getName() + ".tag");
// Does not replace metadata, so we don't have to handle DbgVariableRecords.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4a55109bd2b23..14b151180f6a9 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2434,7 +2434,7 @@ instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
FMFSource = &II;
}
- CallInst *Res;
+ Value *Res;
if (MergeIntoAddendOp)
Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
{P, AddendOp, MulOp0, MulOp1}, FMFSource);
@@ -2457,10 +2457,11 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.replaceInstUsesWith(II, Load);
}
- CallInst *MaskedLoad =
+ Value *MaskedLoad =
IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
Pred, ConstantAggregateZero::get(VecTy));
- MaskedLoad->copyMetadata(II);
+ if (auto *NewI = dyn_cast<Instruction>(MaskedLoad))
+ NewI->copyMetadata(II);
return IC.replaceInstUsesWith(II, MaskedLoad);
}
@@ -2476,9 +2477,10 @@ instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
return IC.eraseInstFromFunction(II);
}
- CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
+ Value *MaskedStore = IC.Builder.CreateMaskedStore(
VecOp, PtrOp, PtrOp->getPointerAlignment(DL), Pred);
- MaskedStore->copyMetadata(II);
+ if (auto *NewI = dyn_cast<Instruction>(MaskedStore))
+ NewI->copyMetadata(II);
return IC.eraseInstFromFunction(II);
}
@@ -2720,7 +2722,7 @@ instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
BasePtr, IndexBase);
- CallInst *MaskedLoad =
+ Value *MaskedLoad =
IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
MaskedLoad->takeName(&II);
return IC.replaceInstUsesWith(II, MaskedLoad);
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index cf4f59bd6c304..600a2aa0cb946 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -160,9 +160,9 @@ bool SVEIntrinsicOpts::coalescePTrueIntrinsicCalls(
auto *MostEncompassingPTrueVTy =
cast<VectorType>(MostEncompassingPTrue->getType());
- auto *ConvertToSVBool = Builder.CreateIntrinsic(
+ auto *ConvertToSVBool = cast<Instruction>(Builder.CreateIntrinsic(
Intrinsic::aarch64_sve_convert_to_svbool, {MostEncompassingPTrueVTy},
- {MostEncompassingPTrue});
+ {MostEncompassingPTrue}));
bool ConvertFromCreated = false;
for (auto *PTrue : PTrues) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
index 9af3b05ff01fa..ad8dfee2a96a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
@@ -73,7 +73,8 @@ static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
IRB.SetInsertPoint(Trm);
- return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {});
+ return cast<Instruction>(
+ IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {}));
}
static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index cf4c2bbc78359..89d6df8fbacc2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -691,8 +691,8 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
// We need to know how many lanes are active within the wavefront, and we do
// this by doing a ballot of active lanes.
Type *const WaveTy = B.getIntNTy(ST.getWavefrontSize());
- CallInst *const Ballot =
- B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
+ CallInst *const Ballot = cast<CallInst>(
+ B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue()));
// We need to know how many lanes are active within the wavefront that are
// below us. If we counted each lane linearly starting from 0, a lane is
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index c5501236edb5b..a07752bd6817f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -2326,7 +2326,8 @@ INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
/// Create a workitem.id.x intrinsic call with range metadata.
CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
- CallInst *Tid = B.CreateIntrinsic(Intrinsic::amdgcn_workitem_id_x, {});
+ CallInst *Tid =
+ cast<CallInst>(B.CreateIntrinsic(Intrinsic::amdgcn_workitem_id_x, {}));
ST.makeLIDRangeMetadata(Tid);
return Tid;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index dd6858a15749e..3dbd81cb793e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -240,7 +240,8 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
Args[ImageDimIntr->DMaskIndex] =
ConstantInt::get(DMask->getType(), NewMaskVal);
Args[FragIdIndex] = ConstantInt::get(FragId->getType(), NewFragIdVal);
- CallInst *NewCall = B.CreateIntrinsic(NewIntrinID, OverloadTys, Args);
+ CallInst *NewCall =
+ cast<CallInst>(B.CreateIntrinsic(NewIntrinID, OverloadTys, Args));
LLVM_DEBUG(dbgs() << "Optimize: " << *NewCall << "\n");
NewCalls.push_back(NewCall);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index b17ccef685b21..90a7790b48aa0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -136,18 +136,20 @@ static std::optional<Instruction *> modifyIntrinsicCall(
// Modify arguments and types
Func(Args, OverloadTys);
- CallInst *NewCall = IC.Builder.CreateIntrinsic(NewIntr, OverloadTys, Args);
- NewCall->takeName(&OldIntr);
- NewCall->copyMetadata(OldIntr);
- if (isa<FPMathOperator>(NewCall))
- NewCall->copyFastMathFlags(&OldIntr);
- // Copy attributes
- AttributeList OldAttrList = OldIntr.getAttributes();
- NewCall->setAttributes(OldAttrList);
+ Value *V = IC.Builder.CreateIntrinsic(NewIntr, OverloadTys, Args);
+ V->takeName(&OldIntr);
+ if (auto *NewCall = dyn_cast<CallInst>(V)) {
+ NewCall->copyMetadata(OldIntr);
+ if (isa<FPMathOperator>(NewCall))
+ NewCall->copyFastMathFlags(&OldIntr);
+ // Copy attributes
+ AttributeList OldAttrList = OldIntr.getAttributes();
+ NewCall->setAttributes(OldAttrList);
+ }
// Erase and replace uses
if (!InstToReplace.getType()->isVoidTy())
- IC.replaceInstUsesWith(InstToReplace, NewCall);
+ IC.replaceInstUsesWith(InstToReplace, V);
bool RemoveOldIntr = &OldIntr != &InstToReplace;
@@ -1773,7 +1775,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Value *Args[] = {SrcLHS, SrcRHS,
ConstantInt::get(CC->getType(), SrcPred)};
- CallInst *NewCall = IC.Builder.CreateIntrinsic(
+ Value *NewCall = IC.Builder.CreateIntrinsic(
NewIID, {II.getType(), SrcLHS->getType()}, Args);
NewCall->takeName(&II);
return IC.replaceInstUsesWith(II, NewCall);
@@ -2211,7 +2213,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Args[0] = Src0;
Args[1] = Src1;
- CallInst *NewII = IC.Builder.CreateIntrinsic(
+ Value *NewII = IC.Builder.CreateIntrinsic(
IID, {Src0->getType(), Src1->getType()}, Args, &II);
NewII->takeName(&II);
return IC.replaceInstUsesWith(II, NewII);
@@ -2253,7 +2255,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Args[1] = Src0;
Args[3] = Src1;
- CallInst *NewII = IC.Builder.CreateIntrinsic(
+ Value *NewII = IC.Builder.CreateIntrinsic(
IID, {II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
Args, &II);
NewII->takeName(&II);
@@ -2403,12 +2405,14 @@ static Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
Args[0] = IC.Builder.CreateShuffleVector(II.getOperand(0), EltMask);
}
- CallInst *NewCall =
+ Value *NewCall =
IC.Builder.CreateIntrinsic(II.getIntrinsicID(), OverloadTys, Args);
NewCall->takeName(&II);
- NewCall->copyMetadata(II);
- AttributeList OldAttrList = II.getAttributes();
- NewCall->setAttributes(OldAttrList);
+ if (auto *CI = dyn_cast<CallInst>(NewCall)) {
+ CI->copyMetadata(II);
+ AttributeList OldAttrList = II.getAttributes();
+ CI->setAttributes(OldAttrList);
+ }
if (IsLoad) {
if (NewNumElts == 1) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 679f7d5d8329f..b0ac4749a4044 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1804,7 +1804,8 @@ Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr,
auto *Call = IRB.CreateIntrinsic(IID, Ty, Args);
copyMetadata(Call, I);
- setAlign(Call, Alignment, Arg ? 1 : 0);
+ if (auto *CI = dyn_cast<CallInst>(Call))
+ setAlign(CI, Alignment, Arg ? 1 : 0);
Call->takeName(I);
insertPostMemOpFence(Order, SSID);
@@ -1874,7 +1875,8 @@ PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) {
{AI.getNewValOperand(), AI.getCompareOperand(), Rsrc,
Off, IRB.getInt32(0), IRB.getInt32(Aux)});
copyMetadata(Call, &AI);
- setAlign(Call, AI.getAlign(), 2);
+ if (auto *CI = dyn_cast<CallInst>(Call))
+ setAlign(CI, AI.getAlign(), 2);
Call->takeName(&AI);
insertPostMemOpFence(Order, SSID);
@@ -2326,7 +2328,7 @@ PtrParts SplitPtrStructs::visitIntrinsicInst(IntrinsicInst &I) {
IID == Intrinsic::amdgcn_load_to_lds
? Intrinsic::amdgcn_raw_ptr_buffer_load_lds
: Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds;
- Instruction *NewLoad = IRB.CreateIntrinsic(
+ Value *NewLoad = IRB.CreateIntrinsic(
NewIntr, {}, {Rsrc, LDSPtr, LoadSize, Off, SOffset, ImmOff, Aux});
copyMetadata(NewLoad, &I);
SplitUsers.insert(&I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index d9b018e3cefa0..791c84e74ac55 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -105,18 +105,22 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// The default cluster barrier expects one signal per workgroup. So we need
// a workgroup barrier first.
if (IsSingleWaveWG) {
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {})
- ->copyMetadata(I);
+ Value *NewV =
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
} else {
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
- CallInst *IsFirst = B.CreateIntrinsic(
+ Value *IsFirst = B.CreateIntrinsic(
B.getInt1Ty(), Intrinsic::amdgcn_s_barrier_signal_isfirst,
{BarrierID_32});
- IsFirst->copyMetadata(I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
- ->copyMetadata(I);
+ if (auto *NewI = dyn_cast<Instruction>(IsFirst))
+ NewI->copyMetadata(I);
+ Value *NewV = B.CreateIntrinsic(
+ B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait, {BarrierID_16});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
Instruction *ThenTerm =
SplitBlockAndInsertIfThen(IsFirst, I.getIterator(), false);
@@ -127,14 +131,16 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// barrier in all waves.
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::CLUSTER);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::CLUSTER);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32})
- ->copyMetadata(I);
+ Value *NewV = B.CreateIntrinsic(
+ B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal, {BarrierID_32});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
B.SetInsertPoint(&I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
- ->copyMetadata(I);
+ NewV = B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
+ {BarrierID_16});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
I.eraseFromParent();
return true;
@@ -160,8 +166,10 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// Down-grade waits, remove split signals.
if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {})
- ->copyMetadata(I);
+ Value *NewV =
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_wave_barrier, {});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
} else if (I.getIntrinsicID() ==
Intrinsic::amdgcn_s_barrier_signal_isfirst) {
// If we're the only wave of the workgroup, we're always first.
@@ -176,12 +184,14 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
// Lower to split barriers.
Value *BarrierID_32 = B.getInt32(AMDGPU::Barrier::WORKGROUP);
Value *BarrierID_16 = B.getInt16(AMDGPU::Barrier::WORKGROUP);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal,
- {BarrierID_32})
- ->copyMetadata(I);
- B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
- {BarrierID_16})
- ->copyMetadata(I);
+ Value *NewV = B.CreateIntrinsic(
+ B.getVoidTy(), Intrinsic::amdgcn_s_barrier_signal, {BarrierID_32});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
+ NewV = B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_s_barrier_wait,
+ {BarrierID_16});
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
I.eraseFromParent();
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index a13011eca5a60..ff81741ebeb27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -221,9 +221,9 @@ static bool lowerKernelArguments(Function &F, const TargetMachine &TM,
if (TotalKernArgSize == 0)
return false;
- CallInst *KernArgSegment =
+ CallInst *KernArgSegment = cast<CallInst>(
Builder.CreateIntrinsic(Intrinsic::amdgcn_kernarg_segment_ptr, {},
- nullptr, F.getName() + ".kernarg.segment");
+ nullptr, F.getName() + ".kernarg.segment"));
KernArgSegment->addRetAttr(Attribute::NonNull);
KernArgSegment->addRetAttr(
Attribute::getWithDereferenceableBytes(Ctx, TotalKernArgSize));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 0981f6becffc4..a52cb84fd929a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -1203,10 +1203,10 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
if (!IsAMDHSA) {
- CallInst *LocalSizeY =
- Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_y, {});
- CallInst *LocalSizeZ =
- Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_z, {});
+ CallInst *LocalSizeY = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_y, {}));
+ CallInst *LocalSizeZ = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::r600_read_local_size_z, {}));
ST.makeLIDRangeMetadata(LocalSizeY);
ST.makeLIDRangeMetadata(LocalSizeZ);
@@ -1248,8 +1248,8 @@ AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
// hsa_signal_t completion_signal; // uint64_t wrapper
// } hsa_kernel_dispatch_packet_t
//
- CallInst *DispatchPtr =
- Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {});
+ CallInst *DispatchPtr = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {}));
DispatchPtr->addRetAttr(Attribute::NoAlias);
DispatchPtr->addRetAttr(Attribute::NonNull);
F.removeFnAttr("amdgpu-no-dispatch-ptr");
@@ -1760,7 +1760,7 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToLDS(
case Intrinsic::objectsize: {
Value *Src = Intr->getOperand(0);
- CallInst *NewCall = Builder.CreateIntrinsic(
+ Value *NewCall = Builder.CreateIntrinsic(
Intrinsic::objectsize,
{Intr->getType(), PointerType::get(Context, AMDGPUAS::LOCAL_ADDRESS)},
{Src, Intr->getOperand(1), Intr->getOperand(2), Intr->getOperand(3)});
@@ -1791,9 +1791,11 @@ void AMDGPUPromoteAllocaImpl::finishDeferredAllocaToLDSPromotion(
ID, MI->getRawDest(), MI->getDestAlign(), MI->getRawSource(),
MI->getSourceAlign(), MI->getLength(), MI->isVolatile());
- for (unsigned I = 0; I != 2; ++I) {
- if (uint64_t Bytes = Intr->getParamDereferenceableBytes(I)) {
- B->addDereferenceableParamAttr(I, Bytes);
+ if (auto *CI = dyn_cast<CallInst>(B)) {
+ for (unsigned I = 0; I != 2; ++I) {
+ if (uint64_t Bytes = Intr->getParamDereferenceableBytes(I)) {
+ CI->addDereferenceableParamAttr(I, Bytes);
+ }
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6cc04c021d0b0..e1826f7823598 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20724,8 +20724,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
Value *LoadedShared = nullptr;
if (FullFlatEmulation) {
- CallInst *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
- {Addr}, nullptr, "is.shared");
+ Value *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
+ {Addr}, nullptr, "is.shared");
Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
Builder.SetInsertPoint(SharedBB);
Value *CastToLocal = Builder.CreateAddrSpaceCast(
@@ -20740,8 +20740,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
Builder.SetInsertPoint(CheckPrivateBB);
}
- CallInst *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
- {Addr}, nullptr, "is.private");
+ Value *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
+ {Addr}, nullptr, "is.private");
Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB);
Builder.SetInsertPoint(PrivateBB);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index e08a4f5ebda08..76ffa1be1dd8e 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -21403,7 +21403,8 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
Builder.getInt32(0), Builder.getInt32(7),
Builder.getInt32(10), Builder.getInt32(5)};
- return Builder.CreateIntrinsic(Intrinsic::arm_mcr, args);
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_mcr, args));
} else {
// Instead of using barriers, atomic accesses on these subtargets use
// libcalls.
@@ -21413,7 +21414,8 @@ Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder,
// Only a full system barrier exists in the M-class architectures.
Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
Constant *CDomain = Builder.getInt32(Domain);
- return Builder.CreateIntrinsic(Intrinsic::arm_dmb, CDomain);
+ return cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::arm_dmb, CDomain));
}
}
@@ -21692,11 +21694,12 @@ Value *ARMTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
Type *Tys[] = { Addr->getType() };
Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
- CallInst *CI = Builder.CreateIntrinsic(Int, Tys, Addr);
+ Value *Res = Builder.CreateIntrinsic(Int, Tys, Addr);
- CI->addParamAttr(
- 0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
- return Builder.CreateTruncOrBitCast(CI, ValueTy);
+ if (auto *CI = dyn_cast<CallInst>(Res))
+ CI->addParamAttr(
+ 0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
+ return Builder.CreateTruncOrBitCast(Res, ValueTy);
}
void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
@@ -21896,7 +21899,7 @@ bool ARMTargetLowering::lowerInterleavedLoad(
BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
VecTy->getNumElements() * Factor);
- CallInst *VldN = createLoadIntrinsic(BaseAddr);
+ Value *VldN = createLoadIntrinsic(BaseAddr);
// Replace uses of each shufflevector with the corresponding vector loaded
// by ldN.
diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
index 869436a7e4964..5798f11bda16a 100644
--- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
+++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
@@ -103,43 +103,41 @@ class MVEGatherScatterLowering : public FunctionPass {
// the other summand.
std::pair<Value *, int64_t> getVarAndConst(Value *Inst, int TypeScale);
- Instruction *lowerGather(IntrinsicInst *I);
+ Value *lowerGather(IntrinsicInst *I);
// Create a gather from a base + vector of offsets
- Instruction *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
- Instruction *&Root,
- IRBuilder<> &Builder);
+ Value *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
+ Instruction *&Root, IRBuilder<> &Builder);
// Create a gather from a vector of pointers
- Instruction *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder, int64_t Increment = 0);
// Create an incrementing gather from a vector of pointers
- Instruction *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
- Instruction *lowerScatter(IntrinsicInst *I);
+ Value *lowerScatter(IntrinsicInst *I);
// Create a scatter to a base + vector of offsets
- Instruction *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
- IRBuilder<> &Builder);
+ Value *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
+ IRBuilder<> &Builder);
// Create a scatter to a vector of pointers
- Instruction *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// Create an incrementing scatter from a vector of pointers
- Instruction *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder,
- int64_t Increment = 0);
+ Value *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment = 0);
// QI gathers and scatters can increment their offsets on their own if
// the increment is a constant value (digit)
- Instruction *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
- IRBuilder<> &Builder);
+ Value *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
+ IRBuilder<> &Builder);
// QI gathers/scatters can increment their offsets on their own if the
// increment is a constant value (digit) - this creates a writeback QI
// gather/scatter
- Instruction *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
- Value *Ptr, unsigned TypeScale,
- IRBuilder<> &Builder);
+ Value *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
+ Value *Ptr, unsigned TypeScale,
+ IRBuilder<> &Builder);
// Optimise the base and offsets of the given address
bool optimiseAddress(Value *Address, BasicBlock *BB, LoopInfo *LI);
@@ -397,7 +395,7 @@ MVEGatherScatterLowering::getVarAndConst(Value *Inst, int TypeScale) {
return std::pair<Value *, int64_t>(Summand, Immediate);
}
-Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
+Value *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked gathers: checking transform preconditions\n"
<< *I << "\n");
@@ -423,7 +421,7 @@ Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
Instruction *Root = I;
- Instruction *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Value *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Load)
Load = tryCreateMaskedGatherOffset(I, Ptr, Root, Builder);
if (!Load)
@@ -450,8 +448,10 @@ Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
return Load;
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
- IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherBase(IntrinsicInst *I,
+ Value *Ptr,
+ IRBuilder<> &Builder,
+ int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
LLVM_DEBUG(dbgs() << "masked gathers: loading from vector of pointers\n");
@@ -470,7 +470,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
{Ptr, Builder.getInt32(Increment), Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
auto *Ty = cast<FixedVectorType>(I->getType());
@@ -491,7 +491,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
{Ptr, Builder.getInt32(Increment), Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
+Value *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
IntrinsicInst *I, Value *Ptr, Instruction *&Root, IRBuilder<> &Builder) {
using namespace PatternMatch;
@@ -553,7 +553,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
Root = Extend;
Value *Mask = I->getArgOperand(1);
- Instruction *Load = nullptr;
+ Value *Load = nullptr;
if (!match(Mask, m_One()))
Load = Builder.CreateIntrinsic(
Intrinsic::arm_mve_vldr_gather_offset_predicated,
@@ -574,7 +574,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
return Load;
}
-Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
+Value *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
using namespace PatternMatch;
LLVM_DEBUG(dbgs() << "masked scatters: checking transform preconditions\n"
<< *I << "\n");
@@ -598,7 +598,7 @@ Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
Builder.SetInsertPoint(I);
Builder.SetCurrentDebugLocation(I->getDebugLoc());
- Instruction *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
+ Value *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
if (!Store)
Store = tryCreateMaskedScatterOffset(I, Ptr, Builder);
if (!Store)
@@ -612,7 +612,7 @@ Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
return Store;
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -636,7 +636,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
{Ptr, Builder.getInt32(Increment), Input, Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -658,7 +658,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
{Ptr, Builder.getInt32(Increment), Input, Mask});
}
-Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
+Value *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
using namespace PatternMatch;
Value *Input = I->getArgOperand(0);
@@ -723,7 +723,7 @@ Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
Builder.getInt32(Scale)});
}
-Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
+Value *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
FixedVectorType *Ty;
if (I->getIntrinsicID() == Intrinsic::masked_gather)
@@ -798,7 +798,7 @@ Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
return tryCreateMaskedScatterBase(I, OffsetsIncoming, Builder, Immediate);
}
-Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
+Value *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
IntrinsicInst *I, Value *BasePtr, Value *Offsets, unsigned TypeScale,
IRBuilder<> &Builder) {
// Check whether this gather's offset is incremented by a constant - if so,
@@ -860,8 +860,8 @@ Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
Builder.SetInsertPoint(I);
- Instruction *EndResult;
- Instruction *NewInduction;
+ Value *EndResult;
+ Value *NewInduction;
if (I->getIntrinsicID() == Intrinsic::masked_gather) {
// Build the incrementing gather
Value *Load = tryCreateMaskedGatherBaseWB(I, Phi, Builder, Immediate);
@@ -1281,22 +1281,22 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) {
}
}
for (IntrinsicInst *I : Gathers) {
- Instruction *L = lowerGather(I);
- if (L == nullptr)
+ Value *L = lowerGather(I);
+ if (!isa_and_nonnull<Instruction>(L))
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(L->getParent());
+ SimplifyInstructionsInBlock(cast<Instruction>(L)->getParent());
Changed = true;
}
for (IntrinsicInst *I : Scatters) {
- Instruction *S = lowerScatter(I);
- if (S == nullptr)
+ Value *S = lowerScatter(I);
+ if (!isa_and_nonnull<Instruction>(S))
continue;
// Get rid of any now dead instructions
- SimplifyInstructionsInBlock(S->getParent());
+ SimplifyInstructionsInBlock(cast<Instruction>(S)->getParent());
Changed = true;
}
return Changed;
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b12985d81fa75..4178bac9367cb 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -488,9 +488,8 @@ static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
return aspaceWrapValue(Cache, CI->getFunction(), P);
}
-static Instruction *aspaceMemSet(Intrinsic::ID ID,
- DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemSet(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -515,9 +514,8 @@ static Instruction *aspaceMemSet(Intrinsic::ID ID,
MI->getAAMetadata());
}
-static Instruction *aspaceMemCpy(Intrinsic::ID ID,
- DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemCpy(Intrinsic::ID ID, DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -540,8 +538,7 @@ static Instruction *aspaceMemCpy(Intrinsic::ID ID,
IsVolatile, MI->getAAMetadata());
}
-static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
- CallInst *CI) {
+static Value *aspaceMemMove(DenseMap<Value *, Value *> &Cache, CallInst *CI) {
auto *MI = cast<MemIntrinsic>(CI);
IRBuilder<> B(CI);
@@ -620,7 +617,7 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
if (!IsSet && !IsCpy && !IsMove)
continue;
- Instruction *New;
+ Value *New;
if (IsSet)
New = aspaceMemSet(ID, CastsCache, CI);
else if (IsCpy)
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index 7e02e53239264..cdd083f454823 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -2227,7 +2227,7 @@ bool HexagonLoopIdiomRecognize::processCopyingStore(Loop *CurLoop,
if (Value *Simp = simplifyInstruction(In, {*DL, TLI, DT}))
NumBytes = Simp;
- CallInst *NewCall;
+ Value *NewCall;
if (RuntimeCheck) {
unsigned Threshold = RuntimeMemSizeThreshold;
@@ -2356,7 +2356,8 @@ bool HexagonLoopIdiomRecognize::processCopyingStore(Loop *CurLoop,
RecursivelyDeleteTriviallyDeadInstructions(SI, TLI);
}
- NewCall->setDebugLoc(DLoc);
+ if (auto *CI = dyn_cast<Instruction>(NewCall))
+ CI->setDebugLoc(DLoc);
LLVM_DEBUG(dbgs() << " Formed " << (Overlap ? "memmove: " : "memcpy: ")
<< *NewCall << "\n"
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 80b6545861054..a549cb25798a0 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -803,10 +803,11 @@ auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
if (HVC.isTrue(Mask))
return createSimpleLoad(Builder, ValTy, Ptr, EffA.value(), MDSources);
- Instruction *Load =
+ Value *Load =
Builder.CreateMaskedLoad(ValTy, Ptr, EffA, Mask, PassThru, "mld");
LLVM_DEBUG(dbgs() << "\t[Creating masked Load:] "; Load->dump());
- propagateMetadata(Load, MDSources);
+ if (auto *LI = dyn_cast<Instruction>(Load))
+ propagateMetadata(LI, MDSources);
return Load;
}
@@ -871,9 +872,9 @@ auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
// Mask is unknown
if (!Predicate) {
- Instruction *Store =
- Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
- propagateMetadata(Store, MDSources);
+ Value *Store = Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
+ if (auto *SI = dyn_cast<Instruction>(Store))
+ propagateMetadata(SI, MDSources);
return Store;
}
@@ -2443,7 +2444,7 @@ Value *HvxIdioms::processVGather(Instruction &In) const {
return nullptr;
LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
- Instruction *Gather = nullptr;
+ Value *Gather = nullptr;
Type *NT = HVC.getHvxTy(HVC.getIntTy(32), false);
if (Qual == HvxIdioms::LdSt || Qual == HvxIdioms::Arithmetic) {
// We fully assume the address space is in VTCM. We also assume that all
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 42e65d7c9c847..95f234cf875e4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -243,12 +243,13 @@ static void convertToParamAS(ArrayRef<Use *> OldUses, Value *Param) {
IRBuilder<> Builder(OldInst);
Intrinsic::ID ID = MI->getIntrinsicID();
- CallInst *B = Builder.CreateMemTransferInst(
+ Value *B = Builder.CreateMemTransferInst(
ID, MI->getRawDest(), MI->getDestAlign(), I.NewParam,
MI->getSourceAlign(), MI->getLength(), MI->isVolatile());
- for (unsigned I : {0, 1})
- if (uint64_t Bytes = MI->getParamDereferenceableBytes(I))
- B->addDereferenceableParamAttr(I, Bytes);
+ if (auto *CI = dyn_cast<CallInst>(B))
+ for (unsigned I : {0, 1})
+ if (uint64_t Bytes = MI->getParamDereferenceableBytes(I))
+ CI->addDereferenceableParamAttr(I, Bytes);
return B;
}
}
@@ -291,10 +292,10 @@ static void convertToParamAS(ArrayRef<Use *> OldUses, Value *Param) {
// alignment of the return value based on the alignment of the argument.
static CallInst *createNVVMInternalAddrspaceWrap(IRBuilder<> &IRB,
Argument &Arg) {
- CallInst *ArgInParam = IRB.CreateIntrinsic(
+ CallInst *ArgInParam = cast<CallInst>(IRB.CreateIntrinsic(
Intrinsic::nvvm_internal_addrspace_wrap,
{IRB.getPtrTy(ADDRESS_SPACE_ENTRY_PARAM), Arg.getType()}, &Arg, {},
- Arg.getName() + ".param");
+ Arg.getName() + ".param"));
if (MaybeAlign ParamAlign = Arg.getParamAlign())
ArgInParam->addRetAttr(
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index e43021b1f5379..20f113cdc1f91 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -13062,7 +13062,7 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
// Other Lowering Code
//===----------------------------------------------------------------------===//
-static Instruction *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
+static Value *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
return Builder.CreateIntrinsic(Id, {});
}
@@ -13142,9 +13142,9 @@ Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
if (Ord == AtomicOrdering::SequentiallyConsistent)
- return callIntrinsic(Builder, Intrinsic::ppc_sync);
+ return cast<Instruction>(callIntrinsic(Builder, Intrinsic::ppc_sync));
if (isReleaseOrStronger(Ord))
- return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
+ return cast<Instruction>(callIntrinsic(Builder, Intrinsic::ppc_lwsync));
return nullptr;
}
@@ -13156,10 +13156,10 @@ Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
// http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
// and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
if (isa<LoadInst>(Inst))
- return Builder.CreateIntrinsic(Intrinsic::ppc_cfence, {Inst->getType()},
- {Inst});
+ return cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::ppc_cfence, {Inst->getType()}, {Inst}));
// FIXME: Can use isync for rmw operation.
- return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
+ return cast<Instruction>(callIntrinsic(Builder, Intrinsic::ppc_lwsync));
}
return nullptr;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 7026541cbd53d..55c10533c18db 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -239,7 +239,7 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
if (!isLegalInterleavedAccessType(VTy, MaskFactor, Alignment, AS, DL))
return false;
- CallInst *SegLoad = nullptr;
+ Value *SegLoad = nullptr;
if (MaskFactor < Factor && MaskFactor != 1) {
// Lower to strided segmented load.
unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index f5c462bdc629b..5549e067693be 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -179,9 +179,8 @@ static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
return std::nullopt;
}
-class SPIRVEmitIntrinsics
- : public ModulePass,
- public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
+class SPIRVEmitIntrinsics : public ModulePass,
+ public InstVisitor<SPIRVEmitIntrinsics, Value *> {
const SPIRVTargetMachine &TM;
SPIRVGlobalRegistry *GR = nullptr;
Function *CurrF = nullptr;
@@ -266,7 +265,7 @@ class SPIRVEmitIntrinsics
Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
bool IsPostprocessing);
- void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
+ void replaceMemInstrUses(Instruction *Old, Value *New, IRBuilder<> &B);
void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
bool UnknownElemTypeI8);
@@ -314,8 +313,8 @@ class SPIRVEmitIntrinsics
DenseMap<Function *, CallInst *> Ptrcasts);
void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
- void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
- Instruction *Dest, bool DeleteOld = true);
+ void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src, Value *Dest,
+ bool DeleteOld = true);
void applyDemangledPtrArgTypes(IRBuilder<> &B);
@@ -381,21 +380,21 @@ class SPIRVEmitIntrinsics
public:
static char ID;
SPIRVEmitIntrinsics(const SPIRVTargetMachine &TM) : ModulePass(ID), TM(TM) {}
- Instruction *visitInstruction(Instruction &I) { return &I; }
- Instruction *visitSwitchInst(SwitchInst &I);
- Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
- Instruction *visitIntrinsicInst(IntrinsicInst &I);
- Instruction *visitBitCastInst(BitCastInst &I);
- Instruction *visitInsertElementInst(InsertElementInst &I);
- Instruction *visitExtractElementInst(ExtractElementInst &I);
- Instruction *visitInsertValueInst(InsertValueInst &I);
- Instruction *visitExtractValueInst(ExtractValueInst &I);
- Instruction *visitLoadInst(LoadInst &I);
- Instruction *visitStoreInst(StoreInst &I);
- Instruction *visitAllocaInst(AllocaInst &I);
- Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
- Instruction *visitUnreachableInst(UnreachableInst &I);
- Instruction *visitCallInst(CallInst &I);
+ Value *visitInstruction(Instruction &I) { return &I; }
+ Value *visitSwitchInst(SwitchInst &I);
+ Value *visitGetElementPtrInst(GetElementPtrInst &I);
+ Value *visitIntrinsicInst(IntrinsicInst &I);
+ Value *visitBitCastInst(BitCastInst &I);
+ Value *visitInsertElementInst(InsertElementInst &I);
+ Value *visitExtractElementInst(ExtractElementInst &I);
+ Value *visitInsertValueInst(InsertValueInst &I);
+ Value *visitExtractValueInst(ExtractValueInst &I);
+ Value *visitLoadInst(LoadInst &I);
+ Value *visitStoreInst(StoreInst &I);
+ Value *visitAllocaInst(AllocaInst &I);
+ Value *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Value *visitUnreachableInst(UnreachableInst &I);
+ Value *visitCallInst(CallInst &I);
StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
@@ -528,15 +527,16 @@ void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
Instruction *Src,
- Instruction *Dest,
+ Value *Dest,
bool DeleteOld) {
replaceAllUsesWith(Src, Dest, DeleteOld);
std::string Name = Src->hasName() ? Src->getName().str() : "";
Src->eraseFromParent();
if (!Name.empty()) {
Dest->setName(Name);
- if (Named.insert(Dest).second)
- emitAssignName(Dest, B);
+ if (auto *DestI = dyn_cast<Instruction>(Dest))
+ if (Named.insert(Dest).second)
+ emitAssignName(DestI, B);
}
}
@@ -606,7 +606,7 @@ CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
SmallVector<Value *, 2> Args = {Op, buildMD(getNormalizedPoisonValue(ElemTy)),
B.getInt32(getPointerAddressSpace(OpTy))};
CallInst *PtrCasted =
- B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+ cast<CallInst>(B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args));
GR->buildAssignPtr(B, ElemTy, PtrCasted);
return PtrCasted;
}
@@ -893,7 +893,8 @@ SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
Args.push_back(B.getInt1(GEP.isInBounds()));
Args.push_back(GEP.getOperand(0));
llvm::append_range(Args, Indices);
- auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
+ auto *NewI =
+ cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}));
replaceAllUsesWithAndErase(B, &GEP, NewI);
return NewI;
}
@@ -1546,16 +1547,15 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(
TypeValidated.insert(I);
}
-void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
- Instruction *New,
+void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, Value *New,
IRBuilder<> &B) {
while (!Old->user_empty()) {
auto *U = Old->user_back();
if (isAssignTypeInstr(U)) {
B.SetInsertPoint(U);
SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
- CallInst *AssignCI =
- B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
+ CallInst *AssignCI = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::spv_assign_type, {New->getType()}, Args));
GR->addAssignPtrTypeInstr(New, AssignCI);
U->eraseFromParent();
} else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
@@ -1604,7 +1604,8 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
llvm_unreachable("illegal aggregate intrinsic user");
}
}
- New->copyMetadata(*Old);
+ if (auto *NewI = dyn_cast<Instruction>(New))
+ NewI->copyMetadata(*Old);
Old->eraseFromParent();
}
@@ -1632,7 +1633,8 @@ void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
setInsertPointSkippingPhis(B, I);
BPrepared = true;
}
- auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {});
+ auto *IntrUndef =
+ cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_undef, {}));
I->replaceUsesOfWith(Op, IntrUndef);
AggrConsts[IntrUndef] = AggrUndef;
AggrConstTypes[IntrUndef] = AggrUndef->getType();
@@ -1662,8 +1664,8 @@ void SPIRVEmitIntrinsics::preprocessPoisons(IRBuilder<> &B) {
setInsertPointSkippingPhis(B, &I);
BPrepared = true;
}
- auto *Call =
- B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = Poison;
AggrConstTypes[Call] = OpTy;
Replacement = Call;
@@ -1751,8 +1753,8 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
PrepareInsert();
Type *PoisonTy = Op->getType();
if (PoisonTy->isAggregateType()) {
- auto *Call = B.CreateIntrinsic(Intrinsic::spv_poison,
- {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(B.CreateIntrinsic(
+ Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = cast<PoisonValue>(Op);
AggrConstTypes[Call] = PoisonTy;
Op = Call;
@@ -1763,8 +1765,8 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
Args.push_back(Op);
}
PrepareInsert();
- auto *CI =
- B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args});
+ auto *CI = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args}));
Worklist.push(CI);
I->replaceUsesOfWith(Op, CI);
KeepInst = true;
@@ -1813,7 +1815,7 @@ static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
createSaturatedConversionDecoration(I, B);
}
-Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
+Value *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
if (!Call.isInlineAsm())
return &Call;
@@ -1872,7 +1874,7 @@ void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
createRoundingModeDecoration(FPI, RoundingModeDeco, B);
}
-Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
+Value *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
BasicBlock *ParentBB = I.getParent();
Function *F = ParentBB->getParent();
IRBuilder<> B(ParentBB);
@@ -1887,11 +1889,11 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
BBCases.push_back(Case.getCaseSuccessor());
Args.push_back(BlockAddress::get(F, Case.getCaseSuccessor()));
}
- CallInst *NewI = B.CreateIntrinsic(Intrinsic::spv_switch,
- {I.getOperand(0)->getType()}, {Args});
+ Value *NewV = B.CreateIntrinsic(Intrinsic::spv_switch,
+ {I.getOperand(0)->getType()}, {Args});
// remove switch to avoid its unneeded and undesirable unwrap into branches
// and conditions
- replaceAllUsesWith(&I, NewI);
+ replaceAllUsesWith(&I, NewV);
I.eraseFromParent();
// insert artificial and temporary instruction to preserve valid CFG,
// it will be removed after IR translation pass
@@ -1908,7 +1910,7 @@ static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
return GEP->getNumIndices() > 0 && match(GEP->getOperand(1), m_Zero());
}
-Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
+Value *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
auto *SGEP = dyn_cast<StructuredGEPInst>(&I);
if (!SGEP)
return &I;
@@ -1928,7 +1930,7 @@ Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
+Value *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
@@ -2035,7 +2037,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
+Value *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
Value *Source = I.getOperand(0);
@@ -2319,7 +2321,7 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
}
}
-Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
+Value *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
// If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
// type in LLT and IRTranslator will replace it by the scalar.
if (isVector1(I.getType()))
@@ -2336,8 +2338,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
return NewI;
}
-Instruction *
-SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
+Value *SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
// If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
// type in LLT and IRTranslator will replace it by the scalar.
if (isVector1(I.getVectorOperandType()))
@@ -2353,7 +2354,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
+Value *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
@@ -2366,13 +2367,12 @@ Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
Args.push_back(I.getInsertedValueOperand());
for (auto &Op : I.indices())
Args.push_back(B.getInt32(Op));
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
+Value *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
if (I.getAggregateOperand()->getType()->isAggregateType()) {
@@ -2392,7 +2392,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
+Value *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
if (!I.getType()->isAggregateType())
return &I;
IRBuilder<> B(I.getParent());
@@ -2411,14 +2411,14 @@ Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
IntrinsicId = Intrinsic::spv_atomic_load;
Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
}
- CallInst *NewI =
+ Value *NewI =
B.CreateIntrinsic(IntrinsicId, {I.getOperand(0)->getType()}, Args);
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
+Value *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
if (!AggrStores.contains(&I))
return &I;
IRBuilder<> B(I.getParent());
@@ -2451,14 +2451,15 @@ Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
IntrinsicId = Intrinsic::spv_atomic_store;
Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
}
- auto *NewI = B.CreateIntrinsic(
+ auto *NewV = B.CreateIntrinsic(
IntrinsicId, {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
- NewI->copyMetadata(I);
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ NewI->copyMetadata(I);
I.eraseFromParent();
- return NewI;
+ return NewV;
}
-Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
+Value *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
Value *ArraySize = nullptr;
if (I.isArrayAllocation()) {
const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
@@ -2485,7 +2486,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
return NewI;
}
-Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
+Value *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
assert(I.getType()->isAggregateType() && "Aggregate result is expected");
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
@@ -2544,7 +2545,7 @@ static bool precededByAbortIntrinsic(const UnreachableInst &I,
return false;
}
-Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
+Value *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
if (precededByAbortIntrinsic(I, ST))
return &I;
@@ -2583,7 +2584,7 @@ shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
IRBuilder<> &B) {
auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
- auto *Leaf = B.CreateIntrinsic(Intrinsic::spv_undef, {});
+ auto *Leaf = cast<Instruction>(B.CreateIntrinsic(Intrinsic::spv_undef, {}));
AggrConsts[Leaf] = PoisonValue::get(ElemTy);
AggrConstTypes[Leaf] = ElemTy;
return Leaf;
@@ -2602,8 +2603,8 @@ Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
Elems.push_back(Entry);
}
}
- auto *Composite = B.CreateIntrinsic(Intrinsic::spv_const_composite,
- {B.getInt32Ty()}, Elems);
+ auto *Composite = cast<Instruction>(B.CreateIntrinsic(
+ Intrinsic::spv_const_composite, {B.getInt32Ty()}, Elems));
AggrConsts[Composite] = PoisonValue::get(AggrTy);
AggrConstTypes[Composite] = AggrTy;
return Composite;
@@ -2629,8 +2630,8 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
isa<PoisonValue>(Init) &&
STI->canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze);
if (UsePoison) {
- auto *Call =
- B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {});
+ auto *Call = cast<Instruction>(
+ B.CreateIntrinsic(Intrinsic::spv_poison, {B.getInt32Ty()}, {}));
AggrConsts[Call] = cast<PoisonValue>(Init);
AggrConstTypes[Call] = Init->getType();
InitOp = Call;
@@ -2640,8 +2641,8 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
}
Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType();
Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init;
- auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global,
- {GV.getType(), Ty}, {&GV, Const});
+ auto *InitInst = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::spv_init_global, {GV.getType(), Ty}, {&GV, Const}));
InitInst->setArgOperand(1, InitOp);
}
if (!Init && GV.use_empty())
@@ -3099,7 +3100,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
SmallVector<Value *, 2> Args = {
NewOp, buildMD(getNormalizedPoisonValue(OpElemTy)),
B.getInt32(getPointerAddressSpace(OpTy))};
- CallInst *PtrCasted =
+ Value *PtrCasted =
B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
GR->buildAssignPtr(B, OpElemTy, PtrCasted);
NewOp = PtrCasted;
@@ -3585,8 +3586,10 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
setInsertPointAfterDef(B, I);
// Visitors return either the original/newly created instruction for
// further processing, nullptr otherwise.
- I = visit(*I);
- if (!I)
+ Value *Replacement = visit(*I);
+ if (isa_and_nonnull<Instruction>(Replacement))
+ I = cast<Instruction>(Replacement);
+ else
continue;
// Don't emit intrinsics for convergence operations.
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 343311fb44475..8bff367fc3c38 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -2229,7 +2229,7 @@ void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty,
MDString::get(Ctx, Arg->getName())};
B.CreateIntrinsic(Intrinsic::spv_value_md,
{MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});
- AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {Arg});
+ AssignCI = cast<CallInst>(B.CreateIntrinsic(Intrinsic::fake_use, {Arg}));
} else {
AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
OfType, Arg, {}, B);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
index cb03dcae21ede..5832d88af750c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
@@ -397,8 +397,7 @@ class SPIRVLegalizePointerCastImpl {
SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),
Element->getType(), Int32Ty};
SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)};
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
buildAssignType(B, Vector->getType(), NewI);
return NewI;
}
@@ -410,8 +409,7 @@ class SPIRVLegalizePointerCastImpl {
Type *Int32Ty = Type::getInt32Ty(B.getContext());
SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};
SmallVector<Value *> Args = {Vector, B.getInt32(Index)};
- Instruction *NewI =
- B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
+ Value *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
buildAssignType(B, ElementType, NewI);
return NewI;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 29fb4d1d9ec04..543507d2b92b9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -150,18 +150,18 @@ static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
IsVolatile->setName("isvolatile");
BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F);
IRBuilder<> IRB(EntryBB);
- auto *MemSet = IRB.CreateMemSet(Dest, Val, Len, MSI->getDestAlign(),
- MSI->isVolatile());
+ auto *MemSet = cast<MemSetInst>(IRB.CreateMemSet(
+ Dest, Val, Len, MSI->getDestAlign(), MSI->isVolatile()));
IRB.CreateRetVoid();
- expandMemSetAsLoop(cast<MemSetInst>(MemSet), TTI);
+ expandMemSetAsLoop(MemSet, TTI);
MemSet->eraseFromParent();
break;
}
case Intrinsic::bswap: {
BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F);
IRBuilder<> IRB(EntryBB);
- auto *BSwap = IRB.CreateIntrinsic(Intrinsic::bswap, Intrinsic->getType(),
- F->getArg(0));
+ auto *BSwap = cast<CallInst>(IRB.CreateIntrinsic(
+ Intrinsic::bswap, Intrinsic->getType(), F->getArg(0)));
IRB.CreateRet(BSwap);
IntrinsicLowering IL(M->getDataLayout());
IL.LowerIntrinsicCall(BSwap);
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 2b48986826b6a..02f5057e58828 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -997,7 +997,7 @@ CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
Args.push_back(Arg2);
Args.push_back(buildMD(Arg));
llvm::append_range(Args, Imms);
- return B.CreateIntrinsic(IntrID, {Types}, Args);
+ return cast<CallInst>(B.CreateIntrinsic(IntrID, {Types}, Args));
}
// Return true if there is an opaque pointer type nested in the argument.
diff --git a/llvm/lib/Target/SystemZ/SystemZTDC.cpp b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
index 1894e71792a3f..6948a8bdd9b6d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTDC.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTDC.cpp
@@ -364,7 +364,7 @@ bool SystemZTDCPass::runOnFunction(Function &F) {
// Call the intrinsic, compare result with 0.
IRBuilder<> IRB(I);
Value *MaskVal = ConstantInt::get(Type::getInt64Ty(Ctx), Mask);
- Instruction *TDC =
+ Value *TDC =
IRB.CreateIntrinsic(Intrinsic::s390_tdc, V->getType(), {V, MaskVal});
Value *ICmp = IRB.CreateICmp(CmpInst::ICMP_NE, TDC, Zero32);
I->replaceAllUsesWith(ICmp);
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 4999581489e82..7eab4deb9a14c 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -68,7 +68,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
// intrinsic to the LLVM intrinsic to allow target-independent optimizations.
if (Value *BoolMask = getBoolVecFromMask(Mask, IC.getDataLayout())) {
// The pass-through vector for an x86 masked load is a zero vector.
- CallInst *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
+ Value *NewMaskedLoad = IC.Builder.CreateMaskedLoad(
II.getType(), Ptr, Align(1), BoolMask, ZeroVec);
return IC.replaceInstUsesWith(II, NewMaskedLoad);
}
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index df42d1f4388ba..4be5b80481500 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -505,8 +505,8 @@ static Instruction *createTileStore(Instruction *TileDef, Value *Ptr) {
Value *Stride = Builder.getInt64(64);
std::array<Value *, 5> Args = {Row, Col, Ptr, Stride, TileDef};
- Instruction *TileStore =
- Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, Args);
+ Instruction *TileStore = cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, Args));
return TileStore;
}
@@ -824,11 +824,12 @@ bool X86LowerAMXCast::optimizeAMXCastFromPhi(
// Create tilezero at the end of incoming block.
auto *Block = OldPN->getIncomingBlock(I);
BasicBlock::iterator Iter = Block->getTerminator()->getIterator();
- Instruction *NewInst = Builder.CreateIntrinsic(
- Intrinsic::x86_tilezero_internal, {}, {Row, Col});
+ Instruction *NewInst = cast<Instruction>(Builder.CreateIntrinsic(
+ Intrinsic::x86_tilezero_internal, {}, {Row, Col}));
NewInst->moveBefore(Iter);
- NewInst = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
- {IncValue->getType()}, {NewInst});
+ NewInst = cast<Instruction>(
+ Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
+ {IncValue->getType()}, {NewInst}));
NewInst->moveBefore(Iter);
// Replace InValue with new Value.
OldPN->setIncomingValue(I, NewInst);
diff --git a/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp b/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
index f667187b76ed5..6e90896304b56 100644
--- a/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
+++ b/llvm/lib/Transforms/IPO/FatLTOCleanup.cpp
@@ -79,7 +79,7 @@ static bool cleanUpTypeCheckedLoad(Module &M, Function &CheckedLoadFn,
Value *Offset = I->getOperand(1);
Type *PtrTy = I->getType()->getStructElementType(0);
ConstantInt *True = ConstantInt::getTrue(M.getContext());
- Instruction *Load;
+ Value *Load;
if (IsRelative) {
Load =
IRB.CreateIntrinsic(Intrinsic::load_relative, {Offset->getType()},
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b4686a6a63e86..0f9b300912660 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1634,7 +1634,7 @@ Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
return replaceInstUsesWith(BO, II);
}
- CallInst *NewClass =
+ Value *NewClass =
Builder.CreateIntrinsic(Intrinsic::is_fpclass, {ClassVal0->getType()},
{ClassVal0, Builder.getInt32(NewClassMask)});
return replaceInstUsesWith(BO, NewClass);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 8efd1607780e9..c1ecd3203c261 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1600,7 +1600,7 @@ Value *InstCombinerImpl::foldReversedIntrinsicOperands(IntrinsicInst *II) {
// intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
- Instruction *NewIntrinsic = Builder.CreateIntrinsic(
+ Value *NewIntrinsic = Builder.CreateIntrinsic(
II->getType(), II->getIntrinsicID(), NewArgs, FPI);
return Builder.CreateVectorReverse(NewIntrinsic);
}
@@ -3035,8 +3035,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (ElementCount::isKnownGT(NegatedCount, RetCount)) {
SmallVector<Value *, 5> NewArgs(II->args());
NewArgs[NegatedOpArg] = OpNotNeg;
- Instruction *NewMul =
- Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
+ Value *NewMul = Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);
return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));
}
break;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 7b6d380acffe1..aa0fc6c652126 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -4945,7 +4945,7 @@ Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
if (MulHadOtherUses)
Builder.SetInsertPoint(Mul);
- CallInst *Call = Builder.CreateIntrinsic(
+ Value *Call = Builder.CreateIntrinsic(
Div->getOpcode() == Instruction::UDiv ? Intrinsic::umul_with_overflow
: Intrinsic::smul_with_overflow,
X->getType(), {X, Y}, /*FMFSource=*/nullptr, "mul");
@@ -6739,7 +6739,7 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
MulA = Builder.CreateZExt(A, MulType);
if (WidthB < MulWidth)
MulB = Builder.CreateZExt(B, MulType);
- CallInst *Call =
+ Value *Call =
Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, MulType,
{MulA, MulB}, /*FMFSource=*/nullptr, "umul");
IC.addToWorklist(MulInstr);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index e6481c89c0265..f127edb7d8aaa 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -466,15 +466,16 @@ void PointerReplacer::replace(Instruction *I) {
SrcV = SrcReplace;
IC.Builder.SetInsertPoint(MemCpy);
- auto *NewI = IC.Builder.CreateMemTransferInst(
+ auto *NewV = IC.Builder.CreateMemTransferInst(
MemCpy->getIntrinsicID(), DestV, MemCpy->getDestAlign(), SrcV,
MemCpy->getSourceAlign(), MemCpy->getLength(), MemCpy->isVolatile());
AAMDNodes AAMD = MemCpy->getAAMetadata();
- if (AAMD)
- NewI->setAAMetadata(AAMD);
+ if (auto *NewI = dyn_cast<Instruction>(NewV))
+ if (AAMD)
+ NewI->setAAMetadata(AAMD);
IC.eraseInstFromFunction(*MemCpy);
- WorkMap[MemCpy] = NewI;
+ WorkMap[MemCpy] = NewV;
} else if (auto *ASC = dyn_cast<AddrSpaceCastInst>(I)) {
auto *V = getReplacement(ASC->getPointerOperand());
assert(V && "Operand not replaced");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 17ac09c10f41c..a721d8eca7ac5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -636,7 +636,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
Value *Y, Value *Z) {
InstCombiner::BuilderTy &Builder = IC.Builder;
Value *YZ = Builder.CreateNSWAdd(Y, Z);
- Instruction *NewPow = Builder.CreateIntrinsic(
+ Value *NewPow = Builder.CreateIntrinsic(
Intrinsic::powi, {X->getType(), YZ->getType()}, {X, YZ}, &I);
return NewPow;
@@ -654,7 +654,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
m_Deferred(X)))) {
Constant *One = ConstantInt::get(Y->getType(), 1);
if (willNotOverflowSignedAdd(Y, One, I)) {
- Instruction *NewPow = createPowiExpr(I, *this, X, Y, One);
+ Value *NewPow = createPowiExpr(I, *this, X, Y, One);
return replaceInstUsesWith(I, NewPow);
}
}
@@ -668,7 +668,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
match(Op1, m_AllowReassoc(m_Intrinsic<Intrinsic::powi>(m_Specific(X),
m_Value(Z)))) &&
Y->getType() == Z->getType() && willNotOverflowSignedAdd(Y, Z, I)) {
- Instruction *NewPow = createPowiExpr(I, *this, X, Y, Z);
+ Value *NewPow = createPowiExpr(I, *this, X, Y, Z);
return replaceInstUsesWith(I, NewPow);
}
@@ -681,7 +681,7 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
m_Specific(Op1), m_Value(Y))))) &&
willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
- Instruction *NewPow = createPowiExpr(I, *this, Op1, Y, NegOne);
+ Value *NewPow = createPowiExpr(I, *this, Op1, Y, NegOne);
return replaceInstUsesWith(I, NewPow);
}
@@ -1927,7 +1927,7 @@ Instruction *InstCombinerImpl::foldFDivConstantDivisor(BinaryOperator &I) {
(match(I.getOperand(1), m_PosZeroFP()) ||
(I.hasNoSignedZeros() && match(I.getOperand(1), m_AnyZeroFP())))) {
IRBuilder<> B(&I);
- CallInst *CopySign = B.CreateIntrinsic(
+ Value *CopySign = B.CreateIntrinsic(
Intrinsic::copysign, {C->getType()},
{ConstantFP::getInfinity(I.getType()), I.getOperand(0)}, &I);
CopySign->takeName(&I);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index df9ba225657d3..ab2be55abd3b3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -417,9 +417,10 @@ Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
Value *SelectVal = Builder.CreateSelect(Cond, LdexpVal0, LdexpVal1);
Value *SelectExp = Builder.CreateSelect(Cond, LdexpExp0, LdexpExp1);
- CallInst *NewLdexp = Builder.CreateIntrinsic(
+ Value *NewLdexp = Builder.CreateIntrinsic(
TII->getType(), Intrinsic::ldexp, {SelectVal, SelectExp});
- NewLdexp->setFastMathFlags(FMF);
+ if (auto *NewI = dyn_cast<CallInst>(NewLdexp))
+ NewI->setFastMathFlags(FMF);
return replaceInstUsesWith(SI, NewLdexp);
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 608966a5e9289..5b6b156904612 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3978,7 +3978,8 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
MTI->getRawDest(),
ConstantInt::get(Type::getInt8Ty(MI.getContext()), 0),
MTI->getLength(), MTI->getDestAlign());
- M->copyMetadata(*MTI);
+ if (auto *MI = dyn_cast<Instruction>(M))
+ MI->copyMetadata(*MTI);
}
}
}
diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
index fb7e58f4632ef..03b4730b1be0b 100644
--- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -111,7 +111,7 @@ static Value *getBoundsCheckCond(Value *Ptr, Value *InstVal,
static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB,
std::optional<int8_t> GuardKind) {
if (!DebugTrapBB)
- return IRB.CreateIntrinsic(Intrinsic::trap, {});
+ return cast<CallInst>(IRB.CreateIntrinsic(Intrinsic::trap, {}));
uint64_t ImmArg = GuardKind.has_value()
? GuardKind.value()
@@ -121,8 +121,8 @@ static CallInst *InsertTrap(BuilderTy &IRB, bool DebugTrapBB,
if (ImmArg > 255)
ImmArg = 255;
- return IRB.CreateIntrinsic(Intrinsic::ubsantrap,
- ConstantInt::get(IRB.getInt8Ty(), ImmArg));
+ return cast<CallInst>(IRB.CreateIntrinsic(
+ Intrinsic::ubsantrap, ConstantInt::get(IRB.getInt8Ty(), ImmArg)));
}
static CallInst *InsertCall(BuilderTy &IRB, bool MayReturn, StringRef Name) {
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index c45ec68f3cd07..b3da8c516c72b 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1266,9 +1266,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
removeUnreachableBlocks(F);
MS.initializeCallbacks(*F.getParent(), TLI);
- FnPrologueEnd =
+ FnPrologueEnd = cast<Instruction>(
IRBuilder<>(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt())
- .CreateIntrinsic(Intrinsic::donothing, {});
+ .CreateIntrinsic(Intrinsic::donothing, {}));
if (MS.CompileKernel) {
IRBuilder<> IRB(FnPrologueEnd);
@@ -4642,7 +4642,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// copy them.
ShadowArgs.append(1, IRB.CreateBitCast(SrcShadow, Src->getType()));
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -4697,7 +4697,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ShadowArgs.append(1, SrcShadowPtr);
ShadowArgs.append(1, Mask);
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(), ShadowArgs);
// The AVX masked load intrinsics do not have integer variants. We use the
// floating-point variants, which will happily copy the shadows even if
@@ -4747,8 +4747,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Shadows are integer-ish types but some intrinsics require a
// different (e.g., floating-point) type.
Shadow = IRB.CreateBitCast(Shadow, I.getArgOperand(0)->getType());
- CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {Shadow, I.getArgOperand(1)});
+ Value *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {Shadow, I.getArgOperand(1)});
setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
setOriginForNaryOp(I);
@@ -4781,8 +4781,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// different (e.g., floating-point) type.
AShadow = IRB.CreateBitCast(AShadow, I.getArgOperand(0)->getType());
BShadow = IRB.CreateBitCast(BShadow, I.getArgOperand(2)->getType());
- CallInst *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {AShadow, Idx, BShadow});
+ Value *CI = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {AShadow, Idx, BShadow});
setShadow(&I, IRB.CreateBitCast(CI, getShadowTy(&I)));
setOriginForNaryOp(I);
}
@@ -5394,12 +5394,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Value *XShadow = getShadow(X);
Value *BZeroShadow = getCleanShadow(B);
- CallInst *AShadowXShadow = IRB.CreateIntrinsic(
+ Value *AShadowXShadow = IRB.CreateIntrinsic(
I.getType(), I.getIntrinsicID(), {XShadow, AShadow, BZeroShadow});
- CallInst *AShadowX = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {X, AShadow, BZeroShadow});
- CallInst *XShadowA = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
- {XShadow, A, BZeroShadow});
+ Value *AShadowX = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {X, AShadow, BZeroShadow});
+ Value *XShadowA = IRB.CreateIntrinsic(I.getType(), I.getIntrinsicID(),
+ {XShadow, A, BZeroShadow});
unsigned NumElements = cast<FixedVectorType>(I.getType())->getNumElements();
Value *BShadow = getShadow(B);
@@ -5477,7 +5477,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// The NEON vector load instructions handled by this function all have
// integer variants. It is easier to use those rather than trying to cast
// a struct of vectors of floats into a struct of vectors of integers.
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(getShadowTy(&I), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -5560,7 +5560,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Addr, IRB, OutputShadowTy, Align(1), /*isStore*/ true);
ShadowArgs.append(1, OutputShadowPtr);
- CallInst *CI =
+ Value *CI =
IRB.CreateIntrinsic(IRB.getVoidTy(), I.getIntrinsicID(), ShadowArgs);
setShadow(&I, CI);
@@ -5757,8 +5757,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ShadowArgs.push_back(Arg);
}
- CallInst *CI =
- IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
+ Value *CI = IRB.CreateIntrinsic(I.getType(), shadowIntrinsicID, ShadowArgs);
Value *CombinedShadow = CI;
// Combine the computed shadow with the shadow of trailing args
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7a08d99a9e505..50f3bfb219ccf 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1167,18 +1167,18 @@ bool LoopIdiomRecognize::processLoopStridedStore(
CallInst *NewCall;
if (SplatValue) {
- NewCall = Builder.CreateMemSet(BasePtr, SplatValue, MemsetArg,
- MaybeAlign(StoreAlignment),
- /*isVolatile=*/false, AATags);
+ NewCall = cast<CallInst>(Builder.CreateMemSet(
+ BasePtr, SplatValue, MemsetArg, MaybeAlign(StoreAlignment),
+ /*isVolatile=*/false, AATags));
} else if (ForceMemsetPatternIntrinsic ||
isLibFuncEmittable(M, TLI, LibFunc_memset_pattern16)) {
assert(isa<SCEVConstant>(StoreSizeSCEV) && "Expected constant store size");
- NewCall = Builder.CreateIntrinsic(
+ NewCall = cast<CallInst>(Builder.CreateIntrinsic(
Intrinsic::experimental_memset_pattern,
{DestInt8PtrTy, PatternValue->getType(), IntIdxTy},
{BasePtr, PatternValue, MemsetArg,
- ConstantInt::getFalse(M->getContext())});
+ ConstantInt::getFalse(M->getContext())}));
if (StoreAlignment)
cast<MemSetPatternInst>(NewCall)->setDestAlignment(*StoreAlignment);
NewCall->setAAMetadata(AATags);
@@ -1475,20 +1475,20 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
// by previous checks.
if (!IsAtomic) {
if (UseMemMove)
- NewCall = Builder.CreateMemMove(StoreBasePtr, StoreAlign, LoadBasePtr,
- LoadAlign, NumBytes,
- /*isVolatile=*/false, AATags);
+ NewCall = cast<CallInst>(Builder.CreateMemMove(
+ StoreBasePtr, StoreAlign, LoadBasePtr, LoadAlign, NumBytes,
+ /*isVolatile=*/false, AATags));
else
- NewCall =
+ NewCall = cast<CallInst>(
Builder.CreateMemCpy(StoreBasePtr, StoreAlign, LoadBasePtr, LoadAlign,
- NumBytes, /*isVolatile=*/false, AATags);
+ NumBytes, /*isVolatile=*/false, AATags));
} else {
// Create the call.
// Note that unordered atomic loads/stores are *required* by the spec to
// have an alignment but non-atomic loads/stores may not.
- NewCall = Builder.CreateElementUnorderedAtomicMemCpy(
+ NewCall = cast<CallInst>(Builder.CreateElementUnorderedAtomicMemCpy(
StoreBasePtr, *StoreAlign, LoadBasePtr, *LoadAlign, NumBytes, StoreSize,
- AATags);
+ AATags));
}
NewCall->setDebugLoc(TheStore->getDebugLoc());
@@ -2600,27 +2600,29 @@ bool LoopIdiomRecognize::recognizePopcount() {
return true;
}
-static CallInst *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
- const DebugLoc &DL) {
+static Value *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
+ const DebugLoc &DL) {
Value *Ops[] = {Val};
Type *Tys[] = {Val->getType()};
- CallInst *CI = IRBuilder.CreateIntrinsic(Intrinsic::ctpop, Tys, Ops);
- CI->setDebugLoc(DL);
+ Value *V = IRBuilder.CreateIntrinsic(Intrinsic::ctpop, Tys, Ops);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->setDebugLoc(DL);
- return CI;
+ return V;
}
-static CallInst *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
- const DebugLoc &DL, bool ZeroCheck,
- Intrinsic::ID IID) {
+static Value *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
+ const DebugLoc &DL, bool ZeroCheck,
+ Intrinsic::ID IID) {
Value *Ops[] = {Val, IRBuilder.getInt1(ZeroCheck)};
Type *Tys[] = {Val->getType()};
- CallInst *CI = IRBuilder.CreateIntrinsic(IID, Tys, Ops);
- CI->setDebugLoc(DL);
+ Value *V = IRBuilder.CreateIntrinsic(IID, Tys, Ops);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->setDebugLoc(DL);
- return CI;
+ return V;
}
/// Transform the following loop (Using CTLZ, CTTZ is similar):
@@ -3126,7 +3128,7 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
Value *Mask =
Builder.CreateOr(LowBitMask, BitMask, BitPos->getName() + ".mask");
Value *XMasked = Builder.CreateAnd(X, Mask, X->getName() + ".masked");
- CallInst *XMaskedNumLeadingZeros = Builder.CreateIntrinsic(
+ Value *XMaskedNumLeadingZeros = Builder.CreateIntrinsic(
IntrID, Ty, {XMasked, /*is_zero_poison=*/Builder.getTrue()},
/*FMFSource=*/nullptr, XMasked->getName() + ".numleadingzeros");
Value *XMaskedNumActiveBits = Builder.CreateSub(
@@ -3486,7 +3488,7 @@ bool LoopIdiomRecognize::recognizeShiftUntilZero() {
// Step 1: Compute the loop's final IV value / trip count.
- CallInst *ValNumLeadingZeros = Builder.CreateIntrinsic(
+ Value *ValNumLeadingZeros = Builder.CreateIntrinsic(
IntrID, Ty, {Val, /*is_zero_poison=*/Builder.getFalse()},
/*FMFSource=*/nullptr, Val->getName() + ".numleadingzeros");
Value *ValNumActiveBits = Builder.CreateSub(
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 68671ef4b81aa..962a8df881456 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -472,8 +472,8 @@ Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
// Get the starting pointer of the block.
StartPtr = Range.StartPtr;
- AMemSet = Builder.CreateMemSet(StartPtr, ByteVal, Range.End - Range.Start,
- Range.Alignment);
+ AMemSet = cast<Instruction>(Builder.CreateMemSet(
+ StartPtr, ByteVal, Range.End - Range.Start, Range.Alignment));
AMemSet->mergeDIAssignID(Range.TheStores);
LLVM_DEBUG(dbgs() << "Replace stores:\n"; for (Instruction *SI
@@ -676,12 +676,13 @@ bool MemCpyOptPass::processStoreOfLoad(StoreInst *SI, LoadInst *LI,
Builder.CreateTypeSize(Builder.getInt64Ty(), DL.getTypeStoreSize(T));
Instruction *M;
if (UseMemMove)
- M = Builder.CreateMemMove(SI->getPointerOperand(), SI->getAlign(),
- LI->getPointerOperand(), LI->getAlign(),
- Size);
+ M = cast<Instruction>(Builder.CreateMemMove(
+ SI->getPointerOperand(), SI->getAlign(), LI->getPointerOperand(),
+ LI->getAlign(), Size));
else
- M = Builder.CreateMemCpy(SI->getPointerOperand(), SI->getAlign(),
- LI->getPointerOperand(), LI->getAlign(), Size);
+ M = cast<Instruction>(Builder.CreateMemCpy(
+ SI->getPointerOperand(), SI->getAlign(), LI->getPointerOperand(),
+ LI->getAlign(), Size));
M->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
LLVM_DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI << " => " << *M
@@ -804,8 +805,8 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
return false;
IRBuilder<> Builder(SI);
- auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size,
- SI->getAlign());
+ auto *M = cast<Instruction>(Builder.CreateMemSet(
+ SI->getPointerOperand(), ByteVal, Size, SI->getAlign()));
M->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
@@ -1240,18 +1241,20 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
// example we could be moving from movaps -> movq on x86.
Instruction *NewM;
if (UseMemMove)
- NewM = Builder.CreateMemMove(M->getDest(), M->getDestAlign(), CopySource,
- CopySourceAlign, CopyLength, M->isVolatile());
+ NewM = cast<Instruction>(
+ Builder.CreateMemMove(M->getDest(), M->getDestAlign(), CopySource,
+ CopySourceAlign, CopyLength, M->isVolatile()));
else if (M->isForceInlined())
// llvm.memcpy may be promoted to llvm.memcpy.inline, but the converse is
// never allowed since that would allow the latter to be lowered as a call
// to an external function.
- NewM = Builder.CreateMemCpyInline(M->getDest(), M->getDestAlign(),
- CopySource, CopySourceAlign, CopyLength,
- M->isVolatile());
+ NewM = cast<Instruction>(Builder.CreateMemCpyInline(
+ M->getDest(), M->getDestAlign(), CopySource, CopySourceAlign,
+ CopyLength, M->isVolatile()));
else
- NewM = Builder.CreateMemCpy(M->getDest(), M->getDestAlign(), CopySource,
- CopySourceAlign, CopyLength, M->isVolatile());
+ NewM = cast<Instruction>(
+ Builder.CreateMemCpy(M->getDest(), M->getDestAlign(), CopySource,
+ CopySourceAlign, CopyLength, M->isVolatile()));
NewM->copyMetadata(*M, LLVMContext::MD_DIAssignID);
@@ -1369,9 +1372,9 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
// value profiling data about the 2 sizes.
if (auto *SI = dyn_cast<SelectInst>(MemsetLen))
setExplicitlyUnknownBranchWeightsIfProfiled(*SI, DEBUG_TYPE);
- Instruction *NewMemSet =
+ Instruction *NewMemSet = cast<Instruction>(
Builder.CreateMemSet(Builder.CreatePtrAdd(Dest, SrcSize),
- MemSet->getOperand(1), MemsetLen, Alignment);
+ MemSet->getOperand(1), MemsetLen, Alignment));
assert(isa<MemoryDef>(MSSA->getMemoryAccess(MemCpy)) &&
"MemCpy must be a MemoryDef");
@@ -1495,8 +1498,8 @@ bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
Align = commonAlignment(*Align, -MOffset);
}
- Instruction *NewM =
- Builder.CreateMemSet(DestPtr, MemSet->getOperand(1), CopySize, Align);
+ Instruction *NewM = cast<Instruction>(
+ Builder.CreateMemSet(DestPtr, MemSet->getOperand(1), CopySize, Align));
auto *LastDef = cast<MemoryDef>(MSSA->getMemoryAccess(MemCpy));
auto *NewAccess = MSSAU->createMemoryAccessAfter(NewM, nullptr, LastDef);
MSSAU->insertDef(cast<MemoryDef>(NewAccess), /*RenameUses=*/true);
@@ -1824,8 +1827,9 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
if (Value *ByteVal = isBytewiseValue(GV->getInitializer(),
M->getDataLayout())) {
IRBuilder<> Builder(M);
- Instruction *NewM = Builder.CreateMemSet(
- M->getRawDest(), ByteVal, M->getLength(), M->getDestAlign(), false);
+ Instruction *NewM = cast<Instruction>(
+ Builder.CreateMemSet(M->getRawDest(), ByteVal, M->getLength(),
+ M->getDestAlign(), false));
auto *LastDef = cast<MemoryDef>(MA);
auto *NewAccess =
MSSAU->createMemoryAccessAfter(NewM, nullptr, LastDef);
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index c0977ae1b4c6b..8a2bfa4f385ed 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1830,9 +1830,9 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
// Create the statepoint given all the arguments
GCStatepointInst *Token = nullptr;
if (auto *CI = dyn_cast<CallInst>(Call)) {
- CallInst *SPCall = Builder.CreateGCStatepointCall(
+ CallInst *SPCall = cast<CallInst>(Builder.CreateGCStatepointCall(
StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
- TransitionArgs, DeoptArgs, GCLive, "safepoint_token");
+ TransitionArgs, DeoptArgs, GCLive, "safepoint_token"));
SPCall->setTailCallKind(CI->getTailCallKind());
SPCall->setCallingConv(CI->getCallingConv());
@@ -1907,7 +1907,8 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
Token->setName("statepoint_token");
if (!Call->getType()->isVoidTy() && !Call->use_empty()) {
StringRef Name = Call->hasName() ? Call->getName() : "";
- CallInst *GCResult = Builder.CreateGCResult(Token, Call->getType(), Name);
+ CallInst *GCResult =
+ cast<CallInst>(Builder.CreateGCResult(Token, Call->getType(), Name));
GCResult->setAttributes(
AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex,
Call->getAttributes().getRetAttrs()));
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 811dd373eca94..df08e3bbca3e6 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -3676,20 +3676,22 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
SrcPtr = OurPtr;
SrcAlign = SliceAlign;
}
- CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
- Size, II.isVolatile());
- if (AATags)
- New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
-
- APInt Offset(DL.getIndexTypeSizeInBits(DestPtr->getType()), 0);
- if (IsDest) {
- migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8,
- &II, New, DestPtr, nullptr, DL);
- } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
- DestPtr->stripAndAccumulateConstantOffsets(
- DL, Offset, /*AllowNonInbounds*/ true))) {
- migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8,
- SliceSize * 8, &II, New, DestPtr, nullptr, DL);
+ Value *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign, Size,
+ II.isVolatile());
+ if (auto *NewI = dyn_cast<Instruction>(New)) {
+ if (AATags)
+ NewI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
+
+ APInt Offset(DL.getIndexTypeSizeInBits(DestPtr->getType()), 0);
+ if (IsDest) {
+ migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8,
+ &II, NewI, DestPtr, nullptr, DL);
+ } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
+ DestPtr->stripAndAccumulateConstantOffsets(
+ DL, Offset, /*AllowNonInbounds*/ true))) {
+ migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8,
+ SliceSize * 8, &II, NewI, DestPtr, nullptr, DL);
+ }
}
LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
return false;
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index be186ffbf7e42..0ae75f5272545 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1814,7 +1814,7 @@ static void AddAlignmentAssumptions(CallBase &CB, InlineFunctionInfo &IFI) {
if (getKnownAlignment(ArgVal, DL, &CB, AC, &DT) >= *Alignment)
continue;
- CallInst *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
+ Value *NewAsmp = IRBuilder<>(&CB).CreateAlignmentAssumption(
DL, ArgVal, Alignment->value());
AC->registerAssumption(cast<AssumeInst>(NewAsmp));
}
@@ -1833,14 +1833,16 @@ static void HandleByValArgumentInit(Type *ByValType, Value *Dst, Value *Src,
Align DstAlign = Dst->getPointerAlignment(M->getDataLayout());
// Generate a memcpy with the correct alignments.
- CallInst *CI = Builder.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Size);
+ Value *Memcpy = Builder.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Size);
// The verifier requires that all calls of debug-info-bearing functions
// from debug-info-bearing functions have a debug location (for inlining
// purposes). Assign a dummy location to satisfy the constraint.
- if (!CI->getDebugLoc() && InsertBlock->getParent()->getSubprogram())
- if (DISubprogram *SP = CalledFunc->getSubprogram())
- CI->setDebugLoc(DILocation::get(SP->getContext(), 0, 0, SP));
+ if (auto *CI = dyn_cast<CallInst>(Memcpy)) {
+ if (!CI->getDebugLoc() && InsertBlock->getParent()->getSubprogram())
+ if (DISubprogram *SP = CalledFunc->getSubprogram())
+ CI->setDebugLoc(DILocation::get(SP->getContext(), 0, 0, SP));
+ }
}
/// When inlining a call site that has a byval argument,
@@ -3171,8 +3173,8 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
// code with llvm.stacksave/llvm.stackrestore intrinsics.
if (InlinedFunctionInfo.ContainsDynamicAllocas) {
// Insert the llvm.stacksave.
- CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
- .CreateStackSave("savedstack");
+ Value *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
+ .CreateStackSave("savedstack");
// Insert a call to llvm.stackrestore before any return instructions in the
// inlined function.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index b2e45c3340b9f..d09d6bc57ee11 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2278,9 +2278,9 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
// Get the backedge taken count and truncate or extended to the AR type.
Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty);
- CallInst *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
- {AbsStep, TruncTripCount},
- /*FMFSource=*/nullptr, "mul");
+ Value *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
+ {AbsStep, TruncTripCount},
+ /*FMFSource=*/nullptr, "mul");
Value *MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
Value *OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow");
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 4b21b20beb895..aceec08ea2582 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1773,7 +1773,7 @@ static void hoistConditionalLoadsStores(
// extended for vector types in the future.
assert(!getLoadStoreType(I)->isVectorTy() && "not implemented");
auto *Op0 = I->getOperand(0);
- CallInst *MaskedLoadStore = nullptr;
+ Value *MaskedLoadStore = nullptr;
if (auto *LI = dyn_cast<LoadInst>(I)) {
// Handle Load.
auto *Ty = I->getType();
@@ -1816,7 +1816,8 @@ static void hoistConditionalLoadsStores(
// same. Keep it.
// !annotation: Not impact semantics. Keep it.
if (const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
- MaskedLoadStore->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
+ if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
+ CI->addRangeRetAttr(getConstantRangeFromMetadata(*Ranges));
I->dropUBImplyingAttrsAndUnknownMetadata({LLVMContext::MD_annotation});
// FIXME: DIAssignID is not supported for masked store yet.
// (Verifier::visitDIAssignIDMetadata)
@@ -1824,7 +1825,8 @@ static void hoistConditionalLoadsStores(
I->eraseMetadataIf([](unsigned MDKind, MDNode *Node) {
return Node->getMetadataID() == Metadata::DIAssignIDKind;
});
- MaskedLoadStore->copyMetadata(*I);
+ if (auto *CI = dyn_cast<CallInst>(MaskedLoadStore))
+ CI->copyMetadata(*I);
I->eraseFromParent();
}
}
@@ -5755,7 +5757,7 @@ bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
Value* Cond = BI->getCondition();
assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.");
- CallInst *Assumption;
+ Value *Assumption;
if (BI->getSuccessor(0) == BB) {
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
Builder.CreateBr(BI->getSuccessor(1));
@@ -8916,7 +8918,7 @@ static bool removeUndefIntroducingPredecessor(BasicBlock *BB,
// Preserve guarding condition in assume, because it might not be
// inferrable from any dominating condition.
Value *Cond = BI->getCondition();
- CallInst *Assumption;
+ Value *Assumption;
if (BI->getSuccessor(0) == BB)
Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
else
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 1cbc8d0bef5ae..30a6b6cec2c83 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -353,7 +353,10 @@ static Value *copyFlags(const CallInst &Old, Value *New) {
return New;
}
-static Value *mergeAttributesAndFlags(CallInst *NewCI, const CallInst &Old) {
+static Value *mergeAttributesAndFlags(Value *NewV, const CallInst &Old) {
+ CallInst *NewCI = dyn_cast<CallInst>(NewV);
+ if (!NewCI)
+ return NewV;
NewCI->setAttributes(AttributeList::get(
NewCI->getContext(), {NewCI->getAttributes(), Old.getAttributes()}));
NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(
@@ -726,8 +729,8 @@ Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
// We have enough information to now generate the memcpy call to do the
// copy for us. Make a memcpy to copy the nul byte with align = 1.
- CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
- TLI->getAsSizeT(Len, *CI->getModule()));
+ Value *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
+ TLI->getAsSizeT(Len, *CI->getModule()));
mergeAttributesAndFlags(NewCI, *CI);
return Dst;
}
@@ -757,7 +760,7 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
// We have enough information to now generate the memcpy call to do the
// copy for us. Make a memcpy to copy the nul byte with align = 1.
- CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
+ Value *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
mergeAttributesAndFlags(NewCI, *CI);
return DstEnd;
}
@@ -823,8 +826,8 @@ Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
// Transform strlcpy(D, S, N) to memcpy(D, S, N') where N' is the lower
// bound on strlen(S) + 1 and N, optionally followed by a nul store to
// D[N' - 1] if necessary.
- CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
- TLI->getAsSizeT(NBytes, *CI->getModule()));
+ Value *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
+ TLI->getAsSizeT(NBytes, *CI->getModule()));
mergeAttributesAndFlags(NewCI, *CI);
if (!NulTerm) {
@@ -894,11 +897,12 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
// Transform st{p,r}ncpy(D, "", N) to memset(D, '\0', N) for any N.
Align MemSetAlign =
CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne();
- CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
+ Value *NewV = B.CreateMemSet(Dst, B.getInt8('\0'), Size, MemSetAlign);
AttrBuilder ArgAttrs(CI->getContext(), CI->getAttributes().getParamAttrs(0));
- NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
- CI->getContext(), 0, ArgAttrs));
- copyFlags(*CI, NewCI);
+ if (auto *NewCI = dyn_cast<CallInst>(NewV))
+ NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
+ CI->getContext(), 0, ArgAttrs));
+ copyFlags(*CI, NewV);
return Dst;
}
@@ -921,8 +925,8 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
// st{p,r}ncpy(D, S, N) -> memcpy(align 1 D, align 1 S, N) when both
// S and N are constant.
- CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
- TLI->getAsSizeT(N, *CI->getModule()));
+ Value *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1),
+ TLI->getAsSizeT(N, *CI->getModule()));
mergeAttributesAndFlags(NewCI, *CI);
if (!RetEnd)
return Dst;
@@ -1638,8 +1642,8 @@ Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
return nullptr;
// memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
- CallInst *NewCI = B.CreateMemCpy(CI->getArgOperand(0), Align(1),
- CI->getArgOperand(1), Align(1), Size);
+ Value *NewCI = B.CreateMemCpy(CI->getArgOperand(0), Align(1),
+ CI->getArgOperand(1), Align(1), Size);
mergeAttributesAndFlags(NewCI, *CI);
return CI->getArgOperand(0);
}
@@ -1688,7 +1692,7 @@ Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
Value *Dst = CI->getArgOperand(0);
Value *N = CI->getArgOperand(2);
// mempcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n), x + n
- CallInst *NewCI =
+ Value *NewCI =
B.CreateMemCpy(Dst, Align(1), CI->getArgOperand(1), Align(1), N);
// Propagate attributes, but memcpy has no return value, so make sure that
// any return attributes are compliant.
@@ -1704,8 +1708,8 @@ Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
return nullptr;
// memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
- CallInst *NewCI = B.CreateMemMove(CI->getArgOperand(0), Align(1),
- CI->getArgOperand(1), Align(1), Size);
+ Value *NewCI = B.CreateMemMove(CI->getArgOperand(0), Align(1),
+ CI->getArgOperand(1), Align(1), Size);
mergeAttributesAndFlags(NewCI, *CI);
return CI->getArgOperand(0);
}
@@ -1718,7 +1722,7 @@ Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
// memset(p, v, n) -> llvm.memset(align 1 p, v, n)
Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
- CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
+ Value *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
mergeAttributesAndFlags(NewCI, *CI);
return CI->getArgOperand(0);
}
@@ -2251,7 +2255,7 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l)) {
if (Pow->doesNotAccessMemory()) {
- CallInst *NewExp10 =
+ Value *NewExp10 =
B.CreateIntrinsic(Intrinsic::exp10, {Ty}, {Expo}, Pow, "exp10");
return copyFlags(*Pow, NewExp10);
}
@@ -4394,7 +4398,7 @@ bool FortifiedLibCallSimplifier::isFortifiedCallFoldable(
Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
IRBuilderBase &B) {
if (isFortifiedCallFoldable(CI, 3, 2)) {
- CallInst *NewCI =
+ Value *NewCI =
B.CreateMemCpy(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
Align(1), CI->getArgOperand(2));
mergeAttributesAndFlags(NewCI, *CI);
@@ -4406,7 +4410,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
IRBuilderBase &B) {
if (isFortifiedCallFoldable(CI, 3, 2)) {
- CallInst *NewCI =
+ Value *NewCI =
B.CreateMemMove(CI->getArgOperand(0), Align(1), CI->getArgOperand(1),
Align(1), CI->getArgOperand(2));
mergeAttributesAndFlags(NewCI, *CI);
@@ -4419,8 +4423,8 @@ Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
IRBuilderBase &B) {
if (isFortifiedCallFoldable(CI, 3, 2)) {
Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
- CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val,
- CI->getArgOperand(2), Align(1));
+ Value *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val,
+ CI->getArgOperand(2), Align(1));
mergeAttributesAndFlags(NewCI, *CI);
return CI->getArgOperand(0);
}
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index b5437ad8705bf..ca1c76f047caa 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1148,8 +1148,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
assert(TTI.isLegalMaskedLoad(VecTy, Alignment, AS,
TTI::MaskKind::ConstantMask));
Value *Mask = createMaskForExtraElements(C, cast<FixedVectorType>(VecTy));
- VecInst = Builder.CreateMaskedLoad(
- VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
+ VecInst = cast<Instruction>(Builder.CreateMaskedLoad(
+ VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
} else {
// This can happen due to a chain of redundant loads.
// In this case, just use the element-type, and avoid ExtractElement.
@@ -1240,8 +1240,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
TTI::MaskKind::ConstantMask));
Value *Mask =
createMaskForExtraElements(C, cast<FixedVectorType>(Vec->getType()));
- VecInst = Builder.CreateMaskedStore(
- Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask);
+ VecInst = cast<Instruction>(Builder.CreateMaskedStore(
+ Vec, getLoadStorePointerOperand(C[0].Inst), Alignment, Mask));
} else {
// Chain is in offset order, so C[0] is the instr with the lowest offset,
// i.e. the root of the vector.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5210088ded294..be1987b8b7742 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23672,7 +23672,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
setInsertPointAfterBundle(E);
LoadInst *LI = cast<LoadInst>(VL0);
- Instruction *NewLI;
+ Value *NewLI;
FixedVectorType *StridedLoadTy = nullptr;
Value *PO = LI->getPointerOperand();
if (E->State == TreeEntry::Vectorize) {
@@ -23735,16 +23735,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
static_cast<int>(
DL->getTypeAllocSize(ScalarTy))));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
- auto *Inst = Builder.CreateIntrinsic(
+ auto *Intr = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_load,
{StridedLoadTy, PO->getType(), StrideTy},
{PO, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedLoadEC)),
Builder.getInt32(StridedLoadEC)});
- Inst->addParamAttr(
- /*ArgNo=*/0,
- Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
- NewLI = Inst;
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(
+ /*ArgNo=*/0,
+ Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
+ NewLI = Intr;
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state");
Value *VecPtr = vectorizeOperand(E, 0);
@@ -23794,7 +23795,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
VecValue = FinalShuffle(VecValue, E);
Value *Ptr = SI->getPointerOperand();
- Instruction *ST;
+ Value *ST;
if (E->State == TreeEntry::Vectorize) {
ST = Builder.CreateAlignedStore(VecValue, Ptr, SI->getAlign());
} else {
@@ -23824,16 +23825,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
StrideTy, static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
if (StridedStoreTy != VecTy)
VecValue = Builder.CreateBitOrPointerCast(VecValue, StridedStoreTy);
- auto *Inst = Builder.CreateIntrinsic(
+ auto *Intr = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_store,
{StridedStoreTy, Ptr->getType(), StrideTy},
{VecValue, Ptr, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedStoreEC)),
Builder.getInt32(StridedStoreEC)});
- Inst->addParamAttr(
- /*ArgNo=*/1,
- Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
- ST = Inst;
+ if (auto *CI = dyn_cast<CallInst>(Intr))
+ CI->addParamAttr(
+ /*ArgNo=*/1,
+ Attribute::getWithAlignment(Intr->getContext(), CommonAlignment));
+ ST = Intr;
}
Value *V = PropagateIRFlags(ST);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f78651cf8de6c..4946a42ddb1cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2303,10 +2303,11 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
else
assert(Opcode == Instruction::Add && "only add or sub supported for now");
- auto *HistogramInst = State.Builder.CreateIntrinsic(
+ auto *HistogramV = State.Builder.CreateIntrinsic(
Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
- applyMetadata(*HistogramInst);
+ if (auto *HistogramInst = dyn_cast<Instruction>(HistogramV))
+ applyMetadata(*HistogramInst);
}
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
@@ -4082,7 +4083,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
- CallInst *NewLI;
+ Value *NewLI;
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Addr = State.get(getAddr(), !CreateGather);
Value *Mask = nullptr;
@@ -4099,11 +4100,12 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,
{Addr, Mask, EVL}, nullptr, "vp.op.load");
}
- NewLI->addParamAttr(
- 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- applyMetadata(*NewLI);
- Instruction *Res = NewLI;
- State.set(this, Res);
+ if (auto *CI = dyn_cast<CallInst>(NewLI)) {
+ CI->addParamAttr(
+ 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
+ applyMetadata(*CI);
+ }
+ State.set(this, NewLI);
}
InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
@@ -4146,14 +4148,15 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
Value *StoredVal = State.get(StoredVPValue);
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
- Instruction *NewSI = nullptr;
+ Value *NewSI = nullptr;
if (CreateScatter)
NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
else if (Mask)
NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
else
NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
- applyMetadata(*NewSI);
+ if (auto *I = dyn_cast<Instruction>(NewSI))
+ applyMetadata(*I);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4170,7 +4173,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
- CallInst *NewSI = nullptr;
+ Value *NewSI = nullptr;
Value *StoredVal = State.get(StoredValue);
Value *EVL = State.get(getEVL(), VPLane(0));
Value *Mask = nullptr;
@@ -4189,9 +4192,11 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
Intrinsic::vp_store,
{StoredVal, Addr, Mask, EVL});
}
- NewSI->addParamAttr(
- 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- applyMetadata(*NewSI);
+ if (auto *CI = dyn_cast<CallInst>(NewSI)) {
+ CI->addParamAttr(
+ 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
+ applyMetadata(*CI);
+ }
}
InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
@@ -4356,7 +4361,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(MaskForGaps && "Mask for Gaps is required but it is null");
}
- Instruction *NewLoad;
+ Value *NewLoad;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
Value *PoisonVec = PoisonValue::get(VecTy);
@@ -4366,9 +4371,11 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
} else
NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
Group->getAlign(), "wide.vec");
- applyMetadata(*NewLoad);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewLoad);
+ if (auto *NewI = dyn_cast<Instruction>(NewLoad)) {
+ applyMetadata(*NewI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(NewI);
+ }
ArrayRef<VPRecipeValue *> VPDefs = definedValues();
if (VecTy->isScalableTy()) {
@@ -4461,18 +4468,20 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- Instruction *NewStoreInstr;
+ Value *NewStore;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
- NewStoreInstr = State.Builder.CreateMaskedStore(
- IVec, ResAddr, Group->getAlign(), GroupMask);
+ NewStore = State.Builder.CreateMaskedStore(IVec, ResAddr, Group->getAlign(),
+ GroupMask);
} else
- NewStoreInstr =
+ NewStore =
State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());
- applyMetadata(*NewStoreInstr);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewStoreInstr);
+ if (auto *I = dyn_cast<Instruction>(NewStore)) {
+ applyMetadata(*I);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(I);
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -4540,15 +4549,16 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Vectorize the interleaved load group.
if (isa<LoadInst>(Instr)) {
- CallInst *NewLoad = State.Builder.CreateIntrinsic(
+ Value *NewLoad = State.Builder.CreateIntrinsic(
VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL}, nullptr,
"wide.vp.load");
- NewLoad->addParamAttr(0,
- Attribute::getWithAlignment(Ctx, Group->getAlign()));
+ if (auto *CI = dyn_cast<CallInst>(NewLoad)) {
+ CI->addParamAttr(0, Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*NewLoad);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewLoad);
+ applyMetadata(*CI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(CI);
+ }
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
@@ -4604,15 +4614,17 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Interleave all the smaller vectors into one wider vector.
Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
- CallInst *NewStore =
+ Value *NewStore =
State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,
{IVec, ResAddr, GroupMask, InterleaveEVL});
- NewStore->addParamAttr(1,
- Attribute::getWithAlignment(Ctx, Group->getAlign()));
- applyMetadata(*NewStore);
- // TODO: Also manage existing metadata using VPIRMetadata.
- Group->addMetadata(NewStore);
+ if (auto *CI = dyn_cast<CallInst>(NewStore)) {
+ CI->addParamAttr(1, Attribute::getWithAlignment(Ctx, Group->getAlign()));
+
+ applyMetadata(*CI);
+ // TODO: Also manage existing metadata using VPIRMetadata.
+ Group->addMetadata(CI);
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 99e45bdc8ee21..d49df0ccd0c6f 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1813,7 +1813,7 @@ bool VectorCombine::foldBinopOfReductions(Instruction &I) {
else
VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
- Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
+ Value *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
replaceValue(I, *Rdx);
return true;
}
@@ -4245,14 +4245,15 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (IsPartialReduction)
ReduceInput = Builder.CreateShuffleVector(FinalVecV, ExtractMask);
- CallInst *ReducedResult;
+ Value *ReducedResult;
if (IsFloatReduction) {
Value *Identity = ConstantExpr::getBinOpIdentity(
*CommonBinOp, ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
CommonFMF.noSignedZeros());
ReducedResult = Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy},
{Identity, ReduceInput});
- ReducedResult->setFastMathFlags(CommonFMF);
+ if (auto *ReducedI = dyn_cast<Instruction>(ReducedResult))
+ ReducedI->setFastMathFlags(CommonFMF);
} else {
ReducedResult =
Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy}, {ReduceInput});
@@ -5936,12 +5937,13 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
- CallInst *NewVP =
+ Value *NewVP =
Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
{II->getMemoryPointerParam(), NewMask, NewEVL});
// Preserve the original alignment.
- NewVP->addParamAttrs(
- 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
+ if (auto *NewCI = dyn_cast<CallInst>(NewVP))
+ NewCI->addParamAttrs(
+ 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
replaceValue(*Cast, *NewVP);
return true;
}
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index 6e4b7af2495b6..68afa3039a634 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -1147,11 +1147,12 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
B.CreateStore(B.getInt8(0), Bar);
B.CreateLifetimeEnd(Foo);
- Instruction *LifetimeStart = B.CreateLifetimeStart(Foo);
+ auto *LifetimeStart = cast<Instruction>(B.CreateLifetimeStart(Foo));
Instruction *FooStore = B.CreateStore(B.getInt8(0), Foo);
Instruction *BarStore = B.CreateStore(B.getInt8(0), Bar);
- Instruction *BazMemSet = B.CreateMemSet(Baz, B.getInt8(0), 1, Align(1));
+ auto *BazMemSet =
+ cast<Instruction>(B.CreateMemSet(Baz, B.getInt8(0), 1, Align(1)));
B.CreateRetVoid();
setupAnalyses();
@@ -1516,7 +1517,8 @@ TEST_F(MemorySSATest, TestCallClobber) {
Value *Pointer2 = B.CreatePtrAdd(Pointer1, B.getInt64(1));
Instruction *StorePointer1 = B.CreateStore(B.getInt8(0), Pointer1);
Instruction *StorePointer2 = B.CreateStore(B.getInt8(0), Pointer2);
- Instruction *MemSet = B.CreateMemSet(Pointer2, B.getInt8(0), 1, Align(1));
+ auto *MemSet =
+ cast<Instruction>(B.CreateMemSet(Pointer2, B.getInt8(0), 1, Align(1)));
B.CreateRetVoid();
setupAnalyses();
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index b4f1c97f03aca..97b5c5f32e26a 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -169,17 +169,19 @@ TEST_F(IRBuilderTest, IntrinsicMangling) {
CallInst *Call;
// Mangled return type, no arguments.
- Call = Builder.CreateIntrinsic(Int64Ty, Intrinsic::coro_size, {});
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Int64Ty, Intrinsic::coro_size, {}));
EXPECT_EQ(Call->getCalledFunction()->getName(), "llvm.coro.size.i64");
// Void return type, mangled argument type.
- Call =
- Builder.CreateIntrinsic(VoidTy, Intrinsic::set_loop_iterations, Int64Val);
+ Call = cast<CallInst>(Builder.CreateIntrinsic(
+ VoidTy, Intrinsic::set_loop_iterations, Int64Val));
EXPECT_EQ(Call->getCalledFunction()->getName(),
"llvm.set.loop.iterations.i64");
// Mangled return type and argument type.
- Call = Builder.CreateIntrinsic(Int64Ty, Intrinsic::lround, DoubleVal);
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Int64Ty, Intrinsic::lround, DoubleVal));
EXPECT_EQ(Call->getCalledFunction()->getName(), "llvm.lround.i64.f64");
}
@@ -199,8 +201,9 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
Args.push_back(UndefValue::get(PredTy));
Args.push_back(UndefValue::get(SrcVecTy));
- Call = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_fcvtzs_i32f16, Args,
- nullptr, "aarch64.sve.fcvtzs.i32f16");
+ Call = cast<CallInst>(
+ Builder.CreateIntrinsic(Intrinsic::aarch64_sve_fcvtzs_i32f16, Args,
+ nullptr, "aarch64.sve.fcvtzs.i32f16"));
FTy = Call->getFunctionType();
EXPECT_EQ(FTy->getReturnType(), DstVecTy);
for (unsigned i = 0; i != Args.size(); ++i)
@@ -218,8 +221,9 @@ TEST_F(IRBuilderTest, IntrinsicsWithScalableVectors) {
Args.push_back(UndefValue::get(PredTy));
Args.push_back(UndefValue::get(VecTy));
- Call = Builder.CreateIntrinsic(Intrinsic::masked_load, {VecTy, PtrToVecTy},
- Args, nullptr, "masked.load");
+ Call = cast<CallInst>(Builder.CreateIntrinsic(Intrinsic::masked_load,
+ {VecTy, PtrToVecTy}, Args,
+ nullptr, "masked.load"));
FTy = Call->getFunctionType();
EXPECT_EQ(FTy->getReturnType(), VecTy);
for (unsigned i = 0; i != Args.size(); ++i)
@@ -360,12 +364,12 @@ TEST_F(IRBuilderTest, ConstrainedFP) {
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::experimental_constrained_fmul);
-
+
V = Builder.CreateFDiv(V, V);
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::experimental_constrained_fdiv);
-
+
V = Builder.CreateFRem(V, V);
ASSERT_TRUE(isa<IntrinsicInst>(V));
II = cast<IntrinsicInst>(V);
@@ -459,18 +463,18 @@ TEST_F(IRBuilderTest, ConstrainedFP) {
EXPECT_EQ(RoundingMode::Dynamic, CII->getRoundingMode());
// Now override the defaults.
- Call = Builder.CreateConstrainedFPBinOp(
- Intrinsic::experimental_constrained_fadd, V, V, nullptr, "", nullptr,
- RoundingMode::TowardNegative, fp::ebMayTrap);
+ Call = cast<CallInst>(Builder.CreateConstrainedFPBinOp(
+ Intrinsic::experimental_constrained_fadd, V, V, nullptr, "", nullptr,
+ RoundingMode::TowardNegative, fp::ebMayTrap));
CII = cast<ConstrainedFPIntrinsic>(Call);
EXPECT_EQ(CII->getIntrinsicID(), Intrinsic::experimental_constrained_fadd);
EXPECT_EQ(fp::ebMayTrap, CII->getExceptionBehavior());
EXPECT_EQ(RoundingMode::TowardNegative, CII->getRoundingMode());
// Same as previous test for CreateConstrainedFPIntrinsic
- Call = Builder.CreateConstrainedFPIntrinsic(
+ Call = cast<CallInst>(Builder.CreateConstrainedFPIntrinsic(
Intrinsic::experimental_constrained_fadd, {V->getType()}, {V, V}, nullptr,
- "", nullptr, RoundingMode::TowardNegative, fp::ebMayTrap);
+ "", nullptr, RoundingMode::TowardNegative, fp::ebMayTrap));
CII = cast<ConstrainedFPIntrinsic>(Call);
EXPECT_EQ(CII->getIntrinsicID(), Intrinsic::experimental_constrained_fadd);
EXPECT_EQ(fp::ebMayTrap, CII->getExceptionBehavior());
@@ -534,9 +538,9 @@ TEST_F(IRBuilderTest, Lifetime) {
AllocaInst *Var3 = Builder.CreateAlloca(Builder.getInt8Ty(),
Builder.getInt32(123));
- CallInst *Start1 = Builder.CreateLifetimeStart(Var1);
- CallInst *Start2 = Builder.CreateLifetimeStart(Var2);
- CallInst *Start3 = Builder.CreateLifetimeStart(Var3);
+ CallInst *Start1 = cast<CallInst>(Builder.CreateLifetimeStart(Var1));
+ CallInst *Start2 = cast<CallInst>(Builder.CreateLifetimeStart(Var2));
+ CallInst *Start3 = cast<CallInst>(Builder.CreateLifetimeStart(Var3));
EXPECT_EQ(Start1->getArgOperand(0), Var1);
EXPECT_EQ(Start2->getArgOperand(0), Var2);
@@ -696,7 +700,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
ASSERT_TRUE(isa<Instruction>(F));
FDiv = cast<Instruction>(F);
EXPECT_FALSE(FDiv->hasAllowReciprocal());
-
+
// Try individual flags.
FMF.clear();
FMF.setAllowReciprocal();
@@ -755,7 +759,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
EXPECT_TRUE(FAdd->hasApproxFunc());
EXPECT_TRUE(FAdd->hasAllowContract());
EXPECT_FALSE(FAdd->hasAllowReassoc());
-
+
FMF.setAllowReassoc();
Builder.clearFastMathFlags();
Builder.setFastMathFlags(FMF);
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 87d922d22eaac..2853397628142 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -327,7 +327,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicScalar) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 10));
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 20));
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::umax);
@@ -345,7 +346,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicVector) {
Args.push_back(Constant::getNullValue(RetTy));
Args.push_back(Constant::getNullValue(RetTy));
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::umax, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::umax);
@@ -366,7 +368,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicAddressSpace) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 3)); // locality
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1)); // cache type
- CallInst *CI = Builder.CreateIntrinsic(RetTy, Intrinsic::prefetch, Args);
+ auto *CI =
+ cast<CallInst>(Builder.CreateIntrinsic(RetTy, Intrinsic::prefetch, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::prefetch);
@@ -394,8 +397,8 @@ TEST_F(IntrinsicsTest, IRBuilderCreateIntrinsicVarArg) {
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0)); // NumCallArgs
Args.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0)); // Flags
- CallInst *CI = Builder.CreateIntrinsic(
- RetTy, Intrinsic::experimental_gc_statepoint, Args);
+ auto *CI = cast<CallInst>(Builder.CreateIntrinsic(
+ RetTy, Intrinsic::experimental_gc_statepoint, Args));
ASSERT_NE(CI, nullptr);
EXPECT_EQ(CI->getIntrinsicID(), Intrinsic::experimental_gc_statepoint);
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 8ceeacf132da1..67b9ad2585c9f 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -1114,8 +1114,8 @@ TEST(Local, CanReplaceOperandWithVariable) {
// immarg.
Type *PtrPtr = B.getPtrTy(0);
Value *Alloca = B.CreateAlloca(PtrPtr, (unsigned)0);
- CallInst *GCRoot = B.CreateIntrinsic(
- Intrinsic::gcroot, {Alloca, Constant::getNullValue(PtrPtr)});
+ auto *GCRoot = cast<CallInst>(B.CreateIntrinsic(
+ Intrinsic::gcroot, {Alloca, Constant::getNullValue(PtrPtr)}));
EXPECT_TRUE(canReplaceOperandWithVariable(GCRoot, 0)); // Alloca
EXPECT_FALSE(canReplaceOperandWithVariable(GCRoot, 1));
EXPECT_FALSE(canReplaceOperandWithVariable(GCRoot, 2));
More information about the llvm-commits
mailing list