[llvm] [VPlan] Move call widening decision to VPlan. (NFCI) (PR #195518)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 02:19:14 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/195518
>From 58907fe363f4dce6c4521c799a852a95957b32e8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 3 May 2026 12:26:03 +0100
Subject: [PATCH 1/2] [VPlan] Move call widening decision to VPlan. (NFCI)
This patch adds a new makeCallWideningDecisions transform which converts
Call VPInstructions to VPWidenCallRecipe/VPWidenIntrinsicRecipe/VPReplicateRecipe
depending on their costs.
To compute the costs, static helpers are introduced to re-use the
existing VPlan cost model logic:
* VPWidenIntrinsicRecipe::getIntrinsicCost
* VPReplicateRecipe::computeScalarCallCost
The cost-model logic is still retained; we assert that the decisions
match to make sure we do not miss any edge cases. The legacy logic will
be removed in a follow-up.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 134 ++++---------
.../Transforms/Vectorize/VPRecipeBuilder.h | 14 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 21 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 19 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 79 ++++----
.../Transforms/Vectorize/VPlanTransforms.cpp | 186 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 6 +
.../VPlan/AArch64/call-decisions.ll | 6 +-
.../VPlan/vplan-print-after-all.ll | 1 +
9 files changed, 321 insertions(+), 145 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6535f2dc6c72c..a08f58ec89aac 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1312,6 +1312,12 @@ class LoopVectorizationCostModel {
/// trivially hoistable.
bool shouldConsiderInvariant(Value *Op);
+ /// Returns true if \p I has been forced to be scalarized at \p VF.
+ bool isForcedScalar(Instruction *I, ElementCount VF) const {
+ auto FS = ForcedScalars.find(VF);
+ return FS != ForcedScalars.end() && FS->second.contains(I);
+ }
+
private:
unsigned NumPredStores = 0;
@@ -5848,6 +5854,32 @@ uint64_t VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
return CM.getPredBlockCostDivisor(CostKind, BB);
}
+bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
+ return CM.isScalarWithPredication(I, VF) ||
+ CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
+ (VF.isVector() && CM.isProfitableToScalarize(I, VF));
+}
+
+bool VPCostContext::isMaskRequired(Instruction *I) const {
+ return CM.isMaskRequired(I);
+}
+
+std::optional<VPCostContext::CallWideningKind>
+VPCostContext::getLegacyCallKind(CallInst *CI, ElementCount VF) const {
+ if (VF.isScalar())
+ return std::nullopt;
+ switch (CM.getCallWideningDecision(CI, VF).Kind) {
+ case LoopVectorizationCostModel::CM_Scalarize:
+ return CallWideningKind::Scalarize;
+ case LoopVectorizationCostModel::CM_IntrinsicCall:
+ return CallWideningKind::Intrinsic;
+ case LoopVectorizationCostModel::CM_VectorCall:
+ return CallWideningKind::VectorVariant;
+ default:
+ return std::nullopt;
+ }
+}
+
InstructionCost
LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
VPCostContext &CostCtx) const {
@@ -6491,90 +6523,6 @@ VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
Phi, Start, Step, &Plan.getVF(), IndDesc, I, Flags, VPI->getDebugLoc());
}
-VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
- VFRange &Range) {
- CallInst *CI = cast<CallInst>(VPI->getUnderlyingInstr());
- bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
- [this, CI](ElementCount VF) {
- return CM.isScalarWithPredication(CI, VF);
- },
- Range);
-
- if (IsPredicated)
- return nullptr;
-
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
- if (VPCostContext::isFreeScalarIntrinsic(ID))
- return nullptr;
-
- SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
- VPI->op_begin() + CI->arg_size());
-
- // Is it beneficial to perform intrinsic call compared to lib call?
- bool ShouldUseVectorIntrinsic =
- ID && LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) -> bool {
- return CM.getCallWideningDecision(CI, VF).Kind ==
- LoopVectorizationCostModel::CM_IntrinsicCall;
- },
- Range);
- if (ShouldUseVectorIntrinsic)
- return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, *VPI,
- VPI->getDebugLoc());
-
- Function *Variant = nullptr;
- std::optional<unsigned> MaskPos;
- // Is better to call a vectorized version of the function than to to scalarize
- // the call?
- auto ShouldUseVectorCall = LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) -> bool {
- // The following case may be scalarized depending on the VF.
- // The flag shows whether we can use a usual Call for vectorized
- // version of the instruction.
-
- // If we've found a variant at a previous VF, then stop looking. A
- // vectorized variant of a function expects input in a certain shape
- // -- basically the number of input registers, the number of lanes
- // per register, and whether there's a mask required.
- // We store a pointer to the variant in the VPWidenCallRecipe, so
- // once we have an appropriate variant it's only valid for that VF.
- // This will force a different vplan to be generated for each VF that
- // finds a valid variant.
- if (Variant)
- return false;
- LoopVectorizationCostModel::CallWideningDecision Decision =
- CM.getCallWideningDecision(CI, VF);
- if (Decision.Kind == LoopVectorizationCostModel::CM_VectorCall) {
- Variant = Decision.Variant;
- MaskPos = Decision.MaskPos;
- return true;
- }
-
- return false;
- },
- Range);
- if (ShouldUseVectorCall) {
- if (MaskPos.has_value()) {
- // We have 2 cases that would require a mask:
- // 1) The call needs to be predicated, either due to a conditional
- // in the scalar loop or use of an active lane mask with
- // tail-folding, and we use the appropriate mask for the block.
- // 2) No mask is required for the call instruction, but the only
- // available vector variant at this VF requires a mask, so we
- // synthesize an all-true mask.
- VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
-
- Ops.insert(Ops.begin() + *MaskPos, Mask);
- }
-
- Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
- return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
- VPI->getDebugLoc());
- }
-
- return nullptr;
-}
-
bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
"Instruction should have been handled earlier");
@@ -6760,10 +6708,12 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range) {
assert(!R->isPhi() && "phis must be handled earlier");
// First, check for specific widening recipes that deal with optimizing
- // truncates, calls and memory operations.
+ // truncates and memory operations
+ auto *VPI = cast<VPInstruction>(R);
+ assert(VPI->getOpcode() != Instruction::Call &&
+ "Call should have been handled by makeCallWideningDecisions");
VPRecipeBase *Recipe;
- auto *VPI = cast<VPInstruction>(R);
if (VPI->getOpcode() == Instruction::Trunc &&
(Recipe = tryToOptimizeInductionTruncate(VPI, Range)))
return Recipe;
@@ -6773,9 +6723,6 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
[&](ElementCount VF) { return VF.isScalar(); }, Range))
return nullptr;
- if (VPI->getOpcode() == Instruction::Call)
- return tryToWidenCall(VPI, Range);
-
Instruction *Instr = R->getUnderlyingInstr();
assert(!is_contained({Instruction::Load, Instruction::Store},
VPI->getOpcode()) &&
@@ -6972,7 +6919,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, TLI, Legal, CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, CM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6991,6 +6938,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeCallWideningDecisions, *Plan,
+ Range, RecipeBuilder, CostCtx);
+
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Convert input VPInstructions to widened recipes.
@@ -7000,8 +6950,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
// transformed.
if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
- VPVectorPointerRecipe, VPVectorEndPointerRecipe,
- VPHistogramRecipe>(&R))
+ VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
+ VPVectorEndPointerRecipe, VPHistogramRecipe>(&R))
continue;
auto *VPI = cast<VPInstruction>(&R);
if (!VPI->getUnderlyingValue())
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a84c77d614673..aff84cdbd0cf7 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -17,7 +17,6 @@ namespace llvm {
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
-class TargetLibraryInfo;
struct HistogramInfo;
struct VFRange;
@@ -26,9 +25,6 @@ class VPRecipeBuilder {
/// The VPlan new recipes are added to.
VPlan &Plan;
- /// Target Library Info.
- const TargetLibraryInfo *TLI;
-
/// The legality analysis.
LoopVectorizationLegality *Legal;
@@ -47,21 +43,15 @@ class VPRecipeBuilder {
VPWidenIntOrFpInductionRecipe *
tryToOptimizeInductionTruncate(VPInstruction *VPI, VFRange &Range);
- /// Handle call instructions. If \p VPI can be widened for \p Range.Start,
- /// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be
- /// decreased to ensure same decision from \p Range.Start to \p Range.End.
- VPSingleDefRecipe *tryToWidenCall(VPInstruction *VPI, VFRange &Range);
-
/// Check if \p VPI has an opcode that can be widened and return a
/// VPWidenRecipe if it can. The function should only be called if the
/// cost-model indicates that widening should be performed.
VPWidenRecipe *tryToWiden(VPInstruction *VPI);
public:
- VPRecipeBuilder(VPlan &Plan, const TargetLibraryInfo *TLI,
- LoopVectorizationLegality *Legal,
+ VPRecipeBuilder(VPlan &Plan, LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM, VPBuilder &Builder)
- : Plan(Plan), TLI(TLI), Legal(Legal), CM(CM), Builder(Builder) {}
+ : Plan(Plan), Legal(Legal), CM(CM), Builder(Builder) {}
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..f42692847dde2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -70,10 +70,6 @@ class LoopVectorizationCostModel;
struct VPCostContext;
-namespace Intrinsic {
-typedef unsigned ID;
-}
-
using VPlanPtr = std::unique_ptr<VPlan>;
/// \enum UncountableExitStyle
@@ -1951,6 +1947,12 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Produce a widened version of the vector intrinsic.
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
+ /// Compute the cost of a vector intrinsic with \p ID and \p Operands.
+ static InstructionCost
+ computeIntrinsicCost(Intrinsic::ID ID, ArrayRef<const VPValue *> Operands,
+ const VPRecipeWithIRFlags &R, ElementCount VF,
+ VPCostContext &Ctx);
+
/// Return the cost of this vector intrinsic.
LLVM_ABI_FOR_TEST InstructionCost
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
@@ -2021,6 +2023,10 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
+ /// Return the cost of widening a call using the vector function \p Variant.
+ static InstructionCost computeVectorCallCost(Function *Variant,
+ VPCostContext &Ctx);
+
Function *getCalledScalarFunction() const {
return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
}
@@ -3232,6 +3238,13 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
+ /// Return the cost of scalarizing a call to \p CalledFn with argument
+ /// operands \p ArgOps for a given \p VF.
+ static InstructionCost
+ computeScalarCallCost(Function *CalledFn, Type *ResultTy,
+ ArrayRef<const VPValue *> ArgOps, bool IsSingleScalar,
+ ElementCount VF, VPCostContext &Ctx);
+
bool isSingleScalar() const { return IsSingleScalar; }
bool isPredicated() const { return IsPredicated; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 62d7782e097dd..ff4b2d7f32964 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -30,6 +30,7 @@ namespace llvm {
class AssumptionCache;
class BasicBlock;
+class CallInst;
class DominatorTree;
class InnerLoopVectorizer;
class IRBuilderBase;
@@ -41,6 +42,10 @@ class VPRegionBlock;
class VPlan;
class Value;
+namespace Intrinsic {
+typedef unsigned ID;
+}
+
/// Returns a calculation for the total number of elements for a given \p VF.
/// For fixed width vectors this value is a constant, whereas for scalable
/// vectors it is an expression determined at runtime.
@@ -324,6 +329,9 @@ struct VPTransformState {
/// Struct to hold various analysis needed for cost computations.
struct VPCostContext {
+ /// Choice for how to widen a call at a given VF.
+ enum class CallWideningKind { Scalarize, Intrinsic, VectorVariant };
+
const TargetTransformInfo &TTI;
const TargetLibraryInfo &TLI;
VPTypeAnalysis Types;
@@ -356,6 +364,17 @@ struct VPCostContext {
/// Forwards to LoopVectorizationCostModel::getPredBlockCostDivisor.
uint64_t getPredBlockCostDivisor(BasicBlock *BB) const;
+ /// Returns true if \p I is known to be scalarized at \p VF.
+ bool willBeScalarized(Instruction *I, ElementCount VF) const;
+
+ /// Forwards to LoopVectorizationCostModel::isMaskRequired.
+ bool isMaskRequired(Instruction *I) const;
+
+ /// Returns the legacy call widening decision for \p CI at \p VF, or
+ /// std::nullopt if none was recorded. Used only in asserts.
+ std::optional<CallWideningKind> getLegacyCallKind(CallInst *CI,
+ ElementCount VF) const;
+
/// Returns the OperandInfo for \p V, if it is a live-in.
TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d04b5edcfc212..3ed0efc7fc226 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1878,6 +1878,11 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ return computeVectorCallCost(Variant, Ctx);
+}
+
+InstructionCost VPWidenCallRecipe::computeVectorCallCost(Function *Variant,
+ VPCostContext &Ctx) {
return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(),
Variant->getFunctionType()->params(),
Ctx.CostKind);
@@ -1963,12 +1968,9 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
State.set(this, V);
}
-/// Compute the cost for the intrinsic \p ID with \p Operands, produced by \p R.
-static InstructionCost getCostForIntrinsics(Intrinsic::ID ID,
- ArrayRef<const VPValue *> Operands,
- const VPRecipeWithIRFlags &R,
- ElementCount VF,
- VPCostContext &Ctx) {
+InstructionCost VPWidenIntrinsicRecipe::computeIntrinsicCost(
+ Intrinsic::ID ID, ArrayRef<const VPValue *> Operands,
+ const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx) {
Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);
// Skip the reverse operation cost for the mask.
// FIXME: Remove this once redundant mask reverse operations can be eliminated
@@ -2012,7 +2014,7 @@ static InstructionCost getCostForIntrinsics(Intrinsic::ID ID,
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
SmallVector<const VPValue *> ArgOps(operands());
- return getCostForIntrinsics(VectorIntrinsicID, ArgOps, *this, VF, Ctx);
+ return computeIntrinsicCost(VectorIntrinsicID, ArgOps, *this, VF, Ctx);
}
StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
@@ -3436,34 +3438,10 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::Call: {
auto *CalledFn =
cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
-
- SmallVector<const VPValue *> ArgOps(drop_end(operands()));
- SmallVector<Type *, 4> Tys;
- for (const VPValue *ArgOp : ArgOps)
- Tys.push_back(Ctx.Types.inferScalarType(ArgOp));
-
- if (CalledFn->isIntrinsic() &&
- VPCostContext::isFreeScalarIntrinsic(CalledFn->getIntrinsicID())) {
- assert(getCostForIntrinsics(CalledFn->getIntrinsicID(), ArgOps, *this,
- ElementCount::getFixed(1), Ctx) == 0 &&
- "scalarizing intrinsic should be free");
- return InstructionCost(0);
- }
-
Type *ResultTy = Ctx.Types.inferScalarType(this);
- InstructionCost ScalarCallCost =
- Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
- if (isSingleScalar()) {
- if (CalledFn->isIntrinsic())
- ScalarCallCost = std::min(
- ScalarCallCost,
- getCostForIntrinsics(CalledFn->getIntrinsicID(), ArgOps, *this,
- ElementCount::getFixed(1), Ctx));
- return ScalarCallCost;
- }
-
- return ScalarCallCost * VF.getFixedValue() +
- Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);
+ SmallVector<const VPValue *> ArgOps(drop_end(operands()));
+ return computeScalarCallCost(CalledFn, ResultTy, ArgOps, isSingleScalar(),
+ VF, Ctx);
}
case Instruction::Add:
case Instruction::Sub:
@@ -3644,6 +3622,39 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
return Ctx.getLegacyCost(UI, VF);
}
+InstructionCost VPReplicateRecipe::computeScalarCallCost(
+ Function *CalledFn, Type *ResultTy, ArrayRef<const VPValue *> ArgOps,
+ bool IsSingleScalar, ElementCount VF, VPCostContext &Ctx) {
+ SmallVector<Type *, 4> Tys = map_to_vector<4>(
+ ArgOps, [&](const VPValue *Op) { return Ctx.Types.inferScalarType(Op); });
+
+ Intrinsic::ID IntrinID = CalledFn->getIntrinsicID();
+ auto GetIntrinsicCost = [&] {
+ return Ctx.TTI.getIntrinsicInstrCost(
+ IntrinsicCostAttributes(IntrinID, ResultTy, Tys), Ctx.CostKind);
+ };
+
+ if (IntrinID && VPCostContext::isFreeScalarIntrinsic(IntrinID)) {
+ assert(GetIntrinsicCost() == 0 && "scalarizing intrinsic should be free");
+ return InstructionCost(0);
+ }
+
+ InstructionCost ScalarCallCost =
+ Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
+ if (IsSingleScalar) {
+ if (IntrinID)
+ ScalarCallCost = std::min(ScalarCallCost, GetIntrinsicCost());
+ return ScalarCallCost;
+ }
+
+ // Scalarization overhead is undefined for scalable VFs.
+ if (VF.isScalable())
+ return InstructionCost::getInvalid();
+
+ return ScalarCallCost * VF.getFixedValue() +
+ Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReplicateRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dadd9253ad57..ad2acdb956df0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6552,3 +6552,189 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
}
}
}
+
+/// Returns true if \p Info's parameter kinds are compatible with \p Args.
+static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
+ PredicatedScalarEvolution &PSE, const Loop *L) {
+ return all_of(Info.Shape.Parameters, [&](VFParameter Param) {
+ switch (Param.ParamKind) {
+ case VFParamKind::Vector:
+ case VFParamKind::GlobalPredicate:
+ return true;
+ case VFParamKind::OMP_Uniform:
+ return PSE.getSE()->isLoopInvariant(
+ vputils::getSCEVExprForVPValue(Args[Param.ParamPos], PSE, L), L);
+ case VFParamKind::OMP_Linear:
+ return match(vputils::getSCEVExprForVPValue(Args[Param.ParamPos], PSE, L),
+ m_scev_AffineAddRec(
+ m_SCEV(), m_scev_SpecificSInt(Param.LinearStepOrPos),
+ m_SpecificLoop(L)));
+ default:
+ return false;
+ }
+ });
+}
+
+/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
+/// Returns the variant function and the position of its mask parameter
+/// (if any), or {nullptr, std::nullopt}.
+static std::pair<Function *, std::optional<unsigned>>
+findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args, ElementCount VF,
+ bool MaskRequired, PredicatedScalarEvolution &PSE,
+ const Loop *L) {
+ if (CI->isNoBuiltin())
+ return {nullptr, std::nullopt};
+ auto Mappings = VFDatabase::getMappings(*CI);
+ const auto *It = find_if(Mappings, [&](const VFInfo &Info) {
+ return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
+ areVFParamsOk(Info, Args, PSE, L);
+ });
+ if (It == Mappings.end())
+ return {nullptr, std::nullopt};
+ if (Function *VecFunc = CI->getModule()->getFunction(It->VectorName))
+ return {VecFunc, It->getParamIndexForOptionalMask()};
+ return {nullptr, std::nullopt};
+}
+
+namespace {
+/// The outcome of choosing how to widen a call at a given VF.
+struct CallWideningDecision {
+ using KindTy = VPCostContext::CallWideningKind;
+ KindTy Kind = KindTy::Scalarize;
+ /// Set when Kind == VectorVariant.
+ Function *Variant = nullptr;
+ /// Position of the mask parameter for \p Variant, if any.
+ std::optional<unsigned> MaskPos;
+};
+} // namespace
+
+/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
+/// vector intrinsic, and vector library variant.
+static CallWideningDecision decideCallWidening(VPInstruction &VPI,
+ ArrayRef<VPValue *> Ops,
+ ElementCount VF,
+ VPCostContext &CostCtx) {
+ auto *CI = cast<CallInst>(VPI.getUnderlyingInstr());
+ auto *CalledFn = cast<Function>(
+ VPI.getOperand(VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
+ Type *ResultTy = CostCtx.Types.inferScalarType(&VPI);
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ bool MaskRequired = CostCtx.isMaskRequired(CI);
+
+ // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
+ if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
+ return {};
+
+ InstructionCost ScalarCost = VPReplicateRecipe::computeScalarCallCost(
+ CalledFn, ResultTy, Ops,
+ /*IsSingleScalar=*/false, VF, CostCtx);
+
+ auto [VecFunc, MaskPos] =
+ findVectorVariant(CI, Ops, VF, MaskRequired, CostCtx.PSE, CostCtx.L);
+ InstructionCost VecCallCost = InstructionCost::getInvalid();
+ if (VecFunc)
+ VecCallCost = VPWidenCallRecipe::computeVectorCallCost(VecFunc, CostCtx);
+
+ // Prefer the intrinsic if it is at least as cheap as scalarizing and any
+ // available vector variant.
+ if (ID) {
+ InstructionCost IntrinsicCost =
+ VPWidenIntrinsicRecipe::computeIntrinsicCost(ID, Ops, VPI, VF, CostCtx);
+ if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
+ (!VecFunc || VecCallCost >= IntrinsicCost))
+ return {CallWideningDecision::KindTy::Intrinsic, nullptr, std::nullopt};
+ }
+
+ // Otherwise, use a vector library variant when it beats scalarizing.
+ if (VecFunc && ScalarCost >= VecCallCost)
+ return {CallWideningDecision::KindTy::VectorVariant, VecFunc, MaskPos};
+
+ return {};
+}
+
+void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx) {
+ bool IsScalarVPlan = LoopVectorizationPlanner::getDecisionAndClampRange(
+ [](ElementCount VF) { return VF.isScalar(); }, Range);
+
+ SmallVector<VPInstruction *, 8> ToErase;
+ for (VPBasicBlock *VPBB :
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI || !VPI->getUnderlyingValue() ||
+ VPI->getOpcode() != Instruction::Call)
+ continue;
+
+ // Scalar VPlans and known-scalarized calls fall through to replication.
+ auto *CI = cast<CallInst>(VPI->getUnderlyingInstr());
+ bool KeepScalar =
+ IsScalarVPlan ||
+ LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) { return CostCtx.willBeScalarized(CI, VF); },
+ Range);
+
+ VPSingleDefRecipe *Recipe = nullptr;
+ CallWideningDecision Decision;
+ if (!KeepScalar) {
+ SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
+ VPI->op_begin() + CI->arg_size());
+
+ // Pick the cheapest widening at Range.Start, then clamp the range.
+ Decision = decideCallWidening(*VPI, Ops, Range.Start, CostCtx);
+ LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ CallWideningDecision D =
+ decideCallWidening(*VPI, Ops, VF, CostCtx);
+ return D.Kind == Decision.Kind && D.Variant == Decision.Variant;
+ },
+ Range);
+
+ switch (Decision.Kind) {
+ case CallWideningDecision::KindTy::Intrinsic: {
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ Type *ResultTy = CostCtx.Types.inferScalarType(VPI);
+ Recipe = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
+ *VPI, VPI->getDebugLoc());
+ break;
+ }
+ case CallWideningDecision::KindTy::VectorVariant: {
+ if (Decision.MaskPos) {
+ VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
+ Ops.insert(Ops.begin() + *Decision.MaskPos, Mask);
+ }
+ Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
+ Recipe =
+ new VPWidenCallRecipe(VPI->getUnderlyingValue(), Decision.Variant,
+ Ops, *VPI, *VPI, VPI->getDebugLoc());
+ break;
+ }
+ case CallWideningDecision::KindTy::Scalarize:
+ break;
+ }
+ }
+
+ if (!Recipe)
+ Recipe = RecipeBuilder.handleReplication(VPI, Range);
+
+ assert(all_of(Range,
+ [&](ElementCount VF) {
+ Intrinsic::ID IID =
+ getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ if (IID && VPCostContext::isFreeScalarIntrinsic(IID))
+ return true;
+ auto Legacy = CostCtx.getLegacyCallKind(CI, VF);
+ return !Legacy || *Legacy == Decision.Kind;
+ }) &&
+ "VPlan call widening decision must match legacy decision");
+
+ Recipe->insertBefore(VPI);
+ VPI->replaceAllUsesWith(Recipe->getVPSingleValue());
+ ToErase.push_back(VPI);
+ }
+ }
+ for (VPInstruction *VPI : ToErase)
+ VPI->eraseFromParent();
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index d0ce6ce71d80c..8ef6a89e89085 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -544,6 +544,12 @@ struct VPlanTransforms {
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
/// enabled by prior widening of consecutive memory operations for now.
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
+
+ /// Convert call VPInstructions in \p Plan into widened call, vector
+ /// intrinsic or replicate recipes based on a cost comparison via \p CostCtx.
+ static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
index 50c605d8aea62..b8054a3ff472a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; -RUN: opt -mattr=+neon -vector-library=LIBMVEC -passes=inject-tli-mappings,loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S -disable-output -vplan-print-after=makeMemOpWideningDecisions %s 2>&1 | FileCheck %s
+; -RUN: opt -mattr=+neon -vector-library=LIBMVEC -passes=inject-tli-mappings,loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S -disable-output -vplan-print-after=makeCallWideningDecisions %s 2>&1 | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -27,7 +27,7 @@ define void @pick_wide_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
-; CHECK-NEXT: EMIT ir<%call> = call ir<%in>, ir<@acos>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @acos(ir<%in>) (using library function: _ZGVnN2v_acos)
; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
@@ -79,7 +79,7 @@ define void @pick_intrinsic_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
-; CHECK-NEXT: EMIT ir<%call> = call ir<100>, ir<%in>, ir<@llvm.umin.i64>
+; CHECK-NEXT: WIDEN-INTRINSIC ir<%call> = call llvm.umin(ir<100>, ir<%in>)
; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 1fb23a06401d4..9babf327060ee 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -14,6 +14,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeScalarizationDecisions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeCallWideningDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
>From b50eb65bf4b6dd208457bf4de330c142cc803df6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 6 May 2026 10:18:05 +0100
Subject: [PATCH 2/2] !fixu address latest comments, thanks
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 88 ++++++++-----------
.../VPlan/AArch64/call-decisions.ll | 10 +--
3 files changed, 48 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3ed0efc7fc226..a858a758a74b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1878,6 +1878,8 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ assert(getVectorizedTypeVF(Variant->getReturnType()) == VF &&
+ "Variant return type must match VF");
return computeVectorCallCost(Variant, Ctx);
}
@@ -3630,6 +3632,8 @@ InstructionCost VPReplicateRecipe::computeScalarCallCost(
Intrinsic::ID IntrinID = CalledFn->getIntrinsicID();
auto GetIntrinsicCost = [&] {
+ if (!IntrinID)
+ return InstructionCost::getInvalid();
return Ctx.TTI.getIntrinsicInstrCost(
IntrinsicCostAttributes(IntrinID, ResultTy, Tys), Ctx.CostKind);
};
@@ -3642,8 +3646,7 @@ InstructionCost VPReplicateRecipe::computeScalarCallCost(
InstructionCost ScalarCallCost =
Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
if (IsSingleScalar) {
- if (IntrinID)
- ScalarCallCost = std::min(ScalarCallCost, GetIntrinsicCost());
+ ScalarCallCost = std::min(ScalarCallCost, GetIntrinsicCost());
return ScalarCallCost;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ad2acdb956df0..8f08cfdda2887 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6615,6 +6615,11 @@ static CallWideningDecision decideCallWidening(VPInstruction &VPI,
ElementCount VF,
VPCostContext &CostCtx) {
auto *CI = cast<CallInst>(VPI.getUnderlyingInstr());
+
+ // Scalar VFs and calls forced or known to scalarize always replicate.
+ if (VF.isScalar() || CostCtx.willBeScalarized(CI, VF))
+ return {};
+
auto *CalledFn = cast<Function>(
VPI.getOperand(VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
Type *ResultTy = CostCtx.Types.inferScalarType(&VPI);
@@ -6655,69 +6660,52 @@ static CallWideningDecision decideCallWidening(VPInstruction &VPI,
void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx) {
- bool IsScalarVPlan = LoopVectorizationPlanner::getDecisionAndClampRange(
- [](ElementCount VF) { return VF.isScalar(); }, Range);
-
SmallVector<VPInstruction *, 8> ToErase;
- for (VPBasicBlock *VPBB :
- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(
- Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || !VPI->getUnderlyingValue() ||
VPI->getOpcode() != Instruction::Call)
continue;
- // Scalar VPlans and known-scalarized calls fall through to replication.
auto *CI = cast<CallInst>(VPI->getUnderlyingInstr());
- bool KeepScalar =
- IsScalarVPlan ||
- LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) { return CostCtx.willBeScalarized(CI, VF); },
- Range);
+ SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
+ VPI->op_begin() + CI->arg_size());
+
+ CallWideningDecision Decision =
+ decideCallWidening(*VPI, Ops, Range.Start, CostCtx);
+ LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ CallWideningDecision D = decideCallWidening(*VPI, Ops, VF, CostCtx);
+ return D.Kind == Decision.Kind && D.Variant == Decision.Variant;
+ },
+ Range);
VPSingleDefRecipe *Recipe = nullptr;
- CallWideningDecision Decision;
- if (!KeepScalar) {
- SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
- VPI->op_begin() + CI->arg_size());
-
- // Pick the cheapest widening at Range.Start, then clamp the range.
- Decision = decideCallWidening(*VPI, Ops, Range.Start, CostCtx);
- LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) {
- CallWideningDecision D =
- decideCallWidening(*VPI, Ops, VF, CostCtx);
- return D.Kind == Decision.Kind && D.Variant == Decision.Variant;
- },
- Range);
-
- switch (Decision.Kind) {
- case CallWideningDecision::KindTy::Intrinsic: {
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
- Type *ResultTy = CostCtx.Types.inferScalarType(VPI);
- Recipe = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
- *VPI, VPI->getDebugLoc());
- break;
- }
- case CallWideningDecision::KindTy::VectorVariant: {
- if (Decision.MaskPos) {
- VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
- Ops.insert(Ops.begin() + *Decision.MaskPos, Mask);
- }
- Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
- Recipe =
- new VPWidenCallRecipe(VPI->getUnderlyingValue(), Decision.Variant,
- Ops, *VPI, *VPI, VPI->getDebugLoc());
- break;
- }
- case CallWideningDecision::KindTy::Scalarize:
- break;
+ switch (Decision.Kind) {
+ case CallWideningDecision::KindTy::Intrinsic: {
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
+ Type *ResultTy = CostCtx.Types.inferScalarType(VPI);
+ Recipe = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI, *VPI,
+ VPI->getDebugLoc());
+ break;
+ }
+ case CallWideningDecision::KindTy::VectorVariant: {
+ if (Decision.MaskPos) {
+ VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
+ Ops.insert(Ops.begin() + *Decision.MaskPos, Mask);
}
+ Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
+ Recipe =
+ new VPWidenCallRecipe(VPI->getUnderlyingValue(), Decision.Variant,
+ Ops, *VPI, *VPI, VPI->getDebugLoc());
+ break;
}
-
- if (!Recipe)
+ case CallWideningDecision::KindTy::Scalarize:
Recipe = RecipeBuilder.handleReplication(VPI, Range);
+ break;
+ }
assert(all_of(Range,
[&](ElementCount VF) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
index b8054a3ff472a..74156dd8ac8ae 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
@@ -24,11 +24,11 @@ define void @pick_wide_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
+; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
-; CHECK-NEXT: WIDEN-CALL ir<%call> = call @acos(ir<%in>) (using library function: _ZGVnN2v_acos)
-; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @acos(ir<%in>) (using library function: _ZGVnN2v_acos)
+; CHECK-NEXT: CLONE ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -76,11 +76,11 @@ define void @pick_intrinsic_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
+; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
; CHECK-NEXT: WIDEN-INTRINSIC ir<%call> = call llvm.umin(ir<100>, ir<%in>)
-; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
+; CHECK-NEXT: CLONE ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
More information about the llvm-commits
mailing list