[llvm] [LV] Have clear cost model before code-gen (NFC) (PR #217556)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 01:55:03 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/217556
The planner previously held a reference to the LoopVectorizationCostModel, which is only needed while planning and computing the best VF. Change the member to a std::unique_ptr so the planner owns it, and release it via a new clearCostModel() before executing the best plan. This prevents code generation from relying on cost-modeling decisions, surfacing any violations.
This patch also migrates one remaining cost-model lookup during codegen (whether partial alias masks are used) to be VPlan-based (via findIncomingAliasMask)
Note that there is one case remaining that needs migrating to VPlan yet. That is checking if an epilogue is allowed.
>From 1ecaf651408d2adc44df4f80100c28c47bda50b7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 24 Jul 2026 10:20:04 +0100
Subject: [PATCH] [LV] Have clear cost model before code-gen (NFC)
The planner previously held a reference to the LoopVectorizationCostModel,
which is only needed while planning and computing the best VF. Change the
member to a std::unique_ptr so the planner owns it, and release it via a new
clearCostModel() before executing the best plan. This prevents code generation
from relying on cost-modeling decisions, surfacing any violations.
Note that there is one case remaining that needs migrating to VPlan yet.
That is checking if an epilogue is allowed.
---
.../Vectorize/LoopVectorizationPlanner.h | 32 +++--
.../Transforms/Vectorize/LoopVectorize.cpp | 114 +++++++++++-------
2 files changed, 91 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cca1d4dbc5720..e3d62a8a475bf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -863,8 +863,8 @@ class LoopVectorizationPlanner {
/// The legality analysis.
LoopVectorizationLegality *Legal;
- /// The profitability analysis.
- LoopVectorizationCostModel &CM;
+ /// The profitability analysis. Cleared after making cost based decisions.
+ std::unique_ptr<LoopVectorizationCostModel> CM;
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -904,11 +904,20 @@ class LoopVectorizationPlanner {
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel &CM, VFSelectionContext &Config,
- InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
- : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
- Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+ std::unique_ptr<LoopVectorizationCostModel> CM,
+ VFSelectionContext &Config, InterleavedAccessInfo &IAI,
+ PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
+
+ ~LoopVectorizationPlanner();
+
+ /// Return the cost model. Must not be called after clearCostModel().
+ LoopVectorizationCostModel &getCostModel() {
+ assert(CM && "Cost model has already been cleared");
+ return *CM;
+ }
+
+ /// Destroy the cost model.
+ void clearCostModel();
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
@@ -972,9 +981,12 @@ class LoopVectorizationPlanner {
/// \return A VPlan for the most profitable epilogue vectorization, with its
/// VF narrowed to the chosen factor. The returned plan is a duplicate.
/// Returns nullptr if epilogue vectorization is not supported or not
- /// profitable for the loop.
- std::unique_ptr<VPlan>
- selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC);
+ /// profitable for the loop. \p ScalarEpilogueAllowed indicates whether the
+ /// epilogue lowering policy permits creating a scalar epilogue at all.
+ std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
+ ElementCount MainLoopVF,
+ unsigned IC,
+ bool ScalarEpilogueAllowed);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bf0bb98eb75b8..8a1edbe1c6bf4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3143,7 +3143,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
if (VF.isScalar())
continue;
- VPCostContext CostCtx(*TLI, *Plan, CM, Config,
+ VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
/*ReusePrintingSlotTracker=*/true);
precomputeCosts(*Plan, VF, CostCtx);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
@@ -3437,19 +3437,20 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
}
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
- VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
+ VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
+ bool ScalarEpilogueAllowed) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
}
- if (!CM.isEpilogueAllowed()) {
+ if (!ScalarEpilogueAllowed) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
"epilogue is allowed.\n");
return nullptr;
}
- if (CM.maskPartialAliasing()) {
+ if (vputils::findIncomingAliasMask(MainPlan)) {
LLVM_DEBUG(
dbgs()
<< "LEV: Epilogue vectorization not supported with alias masking.\n");
@@ -3638,7 +3639,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Do not interleave tail-folded loops, as the overhead of multiple
// instructions to calculate the predicate is likely not beneficial.
// If an epilogue is not allowed for any other reason, do not interleave.
- if (!CM.isEpilogueAllowed())
+ if (!CM->isEpilogueAllowed())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
@@ -3672,7 +3673,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// then we calculate the cost of VF here.
if (LoopCost == 0) {
if (VF.isScalar())
- LoopCost = CM.expectedCost(VF);
+ LoopCost = CM->expectedCost(VF);
else
LoopCost = cost(Plan, VF, &R);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
@@ -3757,7 +3758,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
auto BestKnownTC =
getSmallBestKnownTC(PSE, OrigLoop,
/*CanUseConstantMax=*/true,
- /*CanExcludeZeroTrips=*/CM.isEpilogueAllowed());
+ /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
// For fixed length VFs treat a scalable trip count as unknown.
if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
@@ -5469,10 +5470,10 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
- CM.collectValuesToIgnore();
- Config.collectElementTypesForWidening(&CM.ValuesToIgnore);
+ CM->collectValuesToIgnore();
+ Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
- FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
+ FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
return;
@@ -5502,20 +5503,20 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
Config.computeMinimalBitwidths();
// Invalidate interleave groups if all blocks of loop will be predicated.
- if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ if (CM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs()
<< "LV: Invalidate all interleaved groups due to fold-tail by masking "
"which requires masked-interleaved support.\n");
- if (CM.InterleaveInfo.invalidateGroups())
+ if (CM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- CM.invalidateCostModelingDecisions();
+ CM->invalidateCostModelingDecisions();
}
- if (CM.foldTailByMasking())
+ if (CM->foldTailByMasking())
Legal->prepareToFoldTailByMasking();
ElementCount MaxUserVF =
@@ -5530,12 +5531,12 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
"VF needs to be a power of two");
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- CM.collectNonVectorizedAndSetWideningDecisions(UserVF);
+ CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
buildVPlans(*VPlan1, UserVF, UserVF);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
- CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
@@ -5565,7 +5566,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM.collectNonVectorizedAndSetWideningDecisions(VF);
+ CM->collectNonVectorizedAndSetWideningDecisions(VF);
}
buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
@@ -5768,7 +5769,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, CM, Config,
+ VPCostContext CostCtx(*TLI, Plan, *CM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -5844,7 +5845,7 @@ LoopVectorizationPlanner::computeBestVF() {
"More than a single plan/VF w/o any plan having scalar VF");
// TODO: Compute scalar cost using VPlan-based cost model.
- InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ InstructionCost ScalarCost = CM->expectedCost(ScalarVF);
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
VectorizationFactor BestFactor = ScalarFactor;
@@ -5915,6 +5916,19 @@ LoopVectorizationPlanner::computeBestVF() {
return {BestFactor, &BestPlan};
}
+LoopVectorizationPlanner::LoopVectorizationPlanner(
+ Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
+ std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
+ InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
+ OptimizationRemarkEmitter *ORE)
+ : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
+ CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+
+LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
+
+void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
+
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT,
@@ -5942,7 +5956,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan, BestVF, VScale);
}
- if (CM.maskPartialAliasing()) {
+ if (vputils::findIncomingAliasMask(BestVPlan)) {
assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
*Legal->getRuntimePointerChecking()->getDiffChecks(),
@@ -6568,8 +6582,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6600,7 +6614,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM.foldTailByMasking())
+ if (CM->foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6629,7 +6643,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (CM.foldTailWithEVL()) {
+ if (CM->foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6639,7 +6653,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = CM.getTailFoldingStyle();
+ TailFoldingStyle Style = CM->getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6678,7 +6692,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
[this](ElementCount VF) {
- return !CM.requiresScalarEpilogue(VF.isVector());
+ return !CM->requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6696,9 +6710,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(CM.get(), VF);
- TailFoldingStyle Style = CM.getTailFoldingStyle();
+ TailFoldingStyle Style = CM->getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6727,7 +6741,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
auto ApplyIG = [IG, this](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- CM.getWideningDecision(IG->getInsertPos(), VF) ==
+ CM->getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6744,7 +6758,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6755,7 +6769,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, CM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6855,7 +6869,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// range for better cost estimation.
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
- if (!CM.foldTailWithEVL()) {
+ if (!CM->foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
@@ -6866,7 +6880,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM.isEpilogueAllowed());
+ InterleaveGroups, CM->isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6883,7 +6897,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (CM.maskPartialAliasing())
+ if (CM->maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -6927,7 +6941,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// Remove the predicated select if the target doesn't want it.
VPValue *V;
- if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
+ if (!CM->usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
@@ -8052,14 +8066,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
- GetBFI, F, IAI, Config);
// Use the planner for vectorization.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
- ORE);
+ LoopVectorizationPlanner LVP(
+ L, LI, DT, TLI, *TTI, &LVL,
+ std::make_unique<LoopVectorizationCostModel>(
+ SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
+ Config, IAI, PSE, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE);
+ getEpilogueTailLowering(LVP.getCostModel(), L, ORE);
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
@@ -8093,11 +8108,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
LVP.emitInvalidCostRemarks(ORE);
- assert((IsInnerLoop || !CM.maskPartialAliasing()) &&
+ assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
"Did not expect to alias-mask outer loop");
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
- CM.maskPartialAliasing());
+ LVP.getCostModel().maskPartialAliasing());
if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8123,7 +8138,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- VPCostContext CostCtx(*TLI, *BestPlanPtr, CM, Config,
+ VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
/*ReusePrintingSlotTracker=*/true);
if (!ForceVectorization &&
!isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
@@ -8206,7 +8221,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
- if (CM.maskPartialAliasing()) {
+ if (LVP.getCostModel().maskPartialAliasing()) {
LLVM_DEBUG(
dbgs()
<< "LV: Not interleaving due to partial aliasing vectorization.\n");
@@ -8283,10 +8298,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// If we decided that it is *legal* to interleave or vectorize the loop, then
// do it.
+ // Whether a scalar epilogue may be created is decided by the epilogue
+ // lowering policy.
+ // TODO: Also move check to be based on VPlan.
+ bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
+
+ // Destroy the cost model before executing any plan, so that code generation
+ // cannot rely on cost-modeling decisions.
+ LVP.clearCostModel();
+
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
std::unique_ptr<VPlan> EpiPlan =
- LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC);
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
More information about the llvm-commits
mailing list