[llvm] [LV][NFC] Change Planner's CM reference to a pointer (PR #214898)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 02:09:07 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/214898
>From 86894d76c6f5962fe0ed19513fa2c8636f8b101c Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Sat, 8 Aug 2026 00:34:39 +0100
Subject: [PATCH] [LV] Convert Planner's CM reference to a pointer
---
.../Vectorize/LoopVectorizationPlanner.h | 14 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 100 +++++++++---------
2 files changed, 61 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..7d51cffb1698f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -854,7 +854,8 @@ class LoopVectorizationPlanner {
LoopVectorizationLegality *Legal;
/// The profitability analysis.
- LoopVectorizationCostModel &CM;
+ LoopVectorizationCostModel *DefaultCM;
+ LoopVectorizationCostModel *EnabledCM;
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -896,11 +897,16 @@ class LoopVectorizationPlanner {
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel &CM, VFSelectionContext &Config,
+ LoopVectorizationCostModel *DefaultCM, VFSelectionContext &Config,
InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
const LoopVectorizeHints &Hints, OptimizationRemarkEmitter *ORE)
- : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
- Config(Config), IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}
+ : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
+ DefaultCM(DefaultCM), Config(Config), IAI(IAI), PSE(PSE), Hints(Hints),
+ ORE(ORE) {
+ enableDefaultCM();
+ }
+
+ void enableDefaultCM() { EnabledCM = DefaultCM; }
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28237c7d037d4..938195452100a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3159,7 +3159,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
if (VF.isScalar())
continue;
- VPCostContext CostCtx(*TLI, *Plan, CM, Config,
+ VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
precomputeCosts(*Plan, VF, CostCtx);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
@@ -3459,13 +3459,13 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
return nullptr;
}
- if (!CM.isEpilogueAllowed()) {
+ if (!EnabledCM->isEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
"epilogue is allowed.\n");
return nullptr;
}
- if (CM.maskPartialAliasing()) {
+ if (EnabledCM->maskPartialAliasing()) {
LLVM_DEBUG(
dbgs()
<< "LEV: Epilogue vectorization not supported with alias masking.\n");
@@ -3511,7 +3511,7 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
return nullptr;
}
- if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
+ if (!EnabledCM->isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
"this loop\n");
return nullptr;
@@ -3655,8 +3655,8 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// overhead of multiple instructions to calculate the predicate is likely
// not beneficial. If an epilogue is not allowed for any other reason,
// do not interleave.
- if (!CM.isEpilogueAllowed() &&
- !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()))
+ if (!EnabledCM->isEpilogueAllowed() && !(EnabledCM->preferTailFoldedLoop() &&
+ EnabledCM->useWideActiveLaneMask()))
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
@@ -3684,14 +3684,14 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (hasFindLastReductionPhi(Plan))
return 1;
- VPRegisterUsage R =
- calculateRegisterUsageForPlan(Plan, {VF}, TTI, CM.ValuesToIgnore)[0];
+ VPRegisterUsage R = calculateRegisterUsageForPlan(
+ Plan, {VF}, TTI, EnabledCM->ValuesToIgnore)[0];
// If we did not calculate the cost for VF (because the user selected the VF)
// then we calculate the cost of VF here.
if (LoopCost == 0) {
if (VF.isScalar())
- LoopCost = CM.expectedCost(VF);
+ LoopCost = EnabledCM->expectedCost(VF);
else
LoopCost = cost(Plan, VF, &R);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
@@ -3773,10 +3773,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Try to get the exact trip count, or an estimate based on profiling data or
// ConstantMax from PSE, failing that.
- auto BestKnownTC =
- getSmallBestKnownTC(PSE, OrigLoop,
- /*CanUseConstantMax=*/true,
- /*CanExcludeZeroTrips=*/CM.isEpilogueAllowed());
+ auto BestKnownTC = getSmallBestKnownTC(
+ PSE, OrigLoop,
+ /*CanUseConstantMax=*/true,
+ /*CanExcludeZeroTrips=*/EnabledCM->isEpilogueAllowed());
// For fixed length VFs treat a scalable trip count as unknown.
if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
@@ -5488,10 +5488,10 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
- CM.collectValuesToIgnore();
- Config.collectElementTypesForWidening(&CM.ValuesToIgnore);
+ EnabledCM->collectValuesToIgnore();
+ Config.collectElementTypesForWidening(&EnabledCM->ValuesToIgnore);
- FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
+ FixedScalableVFPair MaxFactors = EnabledCM->computeMaxVF(UserVF, UserIC);
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
return;
@@ -5521,20 +5521,20 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
Config.computeMinimalBitwidths();
// Invalidate interleave groups if all blocks of loop will be predicated.
- if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ if (EnabledCM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs()
<< "LV: Invalidate all interleaved groups due to fold-tail by masking "
"which requires masked-interleaved support.\n");
- if (CM.InterleaveInfo.invalidateGroups())
+ if (EnabledCM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- CM.invalidateCostModelingDecisions();
+ EnabledCM->invalidateCostModelingDecisions();
}
- if (CM.foldTailByMasking())
+ if (EnabledCM->foldTailByMasking())
Legal->prepareToFoldTailByMasking();
ElementCount MaxUserVF =
@@ -5549,12 +5549,12 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
"VF needs to be a power of two");
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- CM.collectNonVectorizedAndSetWideningDecisions(UserVF);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(UserVF);
buildVPlans(*VPlan1, UserVF, UserVF);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
- CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
@@ -5584,7 +5584,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM.collectNonVectorizedAndSetWideningDecisions(VF);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(VF);
}
buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
@@ -5788,7 +5788,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, CM, Config,
+ VPCostContext CostCtx(*TLI, Plan, *EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -5864,7 +5864,7 @@ LoopVectorizationPlanner::computeBestVF() {
"More than a single plan/VF w/o any plan having scalar VF");
// TODO: Compute scalar cost using VPlan-based cost model.
- InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ InstructionCost ScalarCost = EnabledCM->expectedCost(ScalarVF);
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
VectorizationFactor BestFactor = ScalarFactor;
@@ -5888,7 +5888,8 @@ LoopVectorizationPlanner::computeBestVF() {
return Config.shouldConsiderRegPressureForVF(VF);
});
if (ConsiderRegPressure)
- RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
+ RUs = calculateRegisterUsageForPlan(*P, VFs, TTI,
+ EnabledCM->ValuesToIgnore);
for (unsigned I = 0; I < VFs.size(); I++) {
ElementCount VF = VFs[I];
@@ -5947,7 +5948,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF,
- CM.ValuesToIgnore);
+ EnabledCM->ValuesToIgnore);
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
@@ -5962,7 +5963,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan, BestVF, VScale);
}
- if (CM.maskPartialAliasing()) {
+ if (EnabledCM->maskPartialAliasing()) {
assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
*Legal->getRuntimePointerChecking()->getDiffChecks(),
@@ -6587,8 +6588,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6616,7 +6617,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM.foldTailByMasking())
+ if (EnabledCM->foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6645,7 +6646,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (CM.foldTailWithEVL()) {
+ if (EnabledCM->foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6655,7 +6656,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = CM.getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6694,7 +6695,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
[this](ElementCount VF) {
- return !CM.requiresScalarEpilogue(VF.isVector());
+ return !EnabledCM->requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6712,9 +6713,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(EnabledCM, VF);
- TailFoldingStyle Style = CM.getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6743,7 +6744,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
auto ApplyIG = [IG, this](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- CM.getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM->getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6760,7 +6761,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, *EnabledCM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6771,7 +6772,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, CM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6871,7 +6872,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// range for better cost estimation.
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
- if (!CM.foldTailWithEVL()) {
+ if (!EnabledCM->foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
@@ -6882,7 +6883,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM.isEpilogueAllowed());
+ InterleaveGroups, EnabledCM->isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6899,7 +6900,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (CM.maskPartialAliasing())
+ if (EnabledCM->maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -6943,7 +6944,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// Remove the predicated select if the target doesn't want it.
VPValue *V;
- if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
+ if (!EnabledCM->usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
@@ -7122,7 +7123,7 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
assert((!Config.OptForSize ||
- CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
+ EnabledCM->Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
"Cannot SCEV check stride or overflow when optimizing for size");
RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
SCEVCheckBlock, HasBranchWeights);
@@ -7136,7 +7137,7 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
if (Config.OptForSize) {
assert(
- CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
+ EnabledCM->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
"Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.");
ORE->emit([&]() {
@@ -7160,8 +7161,9 @@ bool LoopVectorizationPlanner::requiresScalarEpilogue(VPlan &Plan,
// loop. Must be called before removeBranchOnConst.
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
bool Result = MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
- assert(CM.requiresScalarEpilogue(VF.isVector()) == Result &&
- "CM.requiresScalarEpilogue and the VPlan-based check must agree");
+ assert(
+ EnabledCM->requiresScalarEpilogue(VF.isVector()) == Result &&
+ "EnabledCM->requiresScalarEpilogue and the VPlan-based check must agree");
return Result;
}
@@ -8080,8 +8082,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
GetBFI, F, &Hints, IAI, Config);
// Use the planner for vectorization.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
- Hints, ORE);
+ LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, &CM, Config, IAI,
+ PSE, Hints, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(CM, L, ORE);
More information about the llvm-commits
mailing list