[llvm] [LV] Thread an explicit CM param through VPlan-building/costing helpers (PR #224836)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 16:58:35 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/224836
>From 03a03447ef5359db418f6cd3532b7ddf59ac5095 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 18 Sep 2026 20:13:52 +0100
Subject: [PATCH 1/2] [LV] Thread an explicit cost model through
VPlan-building/costing helpers (NFC)
Parameterize cost(), tryToBuildVPlan1(), tryToBuildVPlan(), and
buildVPlans() with an explicit LoopVectorizationCostModel &EnabledCM
argument instead of always reaching for the planner's own CM member.
This is preparation for tail-folded vector epilogue support, which
will plan against a separate CM from the one used for the main
vector loop. Passing the CM in explicitly lets these functions
be called with whichever CM applies to what's currently being
planned.
All existing call sites still pass *CM, so this is NFC.
---
.../Vectorize/LoopVectorizationPlanner.h | 11 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 81 ++++++++++---------
2 files changed, 51 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 733b8647a06e3..3d2ded9ad0381 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -918,7 +918,8 @@ class LoopVectorizationPlanner {
///
/// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
/// been retired.
- InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
+ InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const;
/// Precompute costs for certain instructions using the legacy cost model. The
/// function is used to bring up the VPlan-based cost model to initially avoid
@@ -1047,7 +1048,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1();
+ VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1057,12 +1058,14 @@ class LoopVectorizationPlanner {
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
/// original scalar loop.
- VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
+ VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
+ LoopVectorizationCostModel &EnabledCM);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 521f6d15a8c18..bb03b82619987 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3722,7 +3722,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (VF.isScalar())
LoopCost = CM->expectedCost(VF);
else
- LoopCost = cost(Plan, VF, &R);
+ LoopCost = cost(Plan, VF, &R, *CM);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
// Loop body is free and there is no need for interleaving.
@@ -5354,7 +5354,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(*CM);
if (!VPlan1)
return;
@@ -5366,7 +5366,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF);
+ buildVPlans(*VPlan1, VF, VF, *CM);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5405,18 +5405,18 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
- buildVPlans(*VPlan1, UserVF, UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, *CM);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
if (UserVF.isScalar() ||
- cost(*VPlans.front(), UserVF, /*RU=*/nullptr).isValid()) {
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid()) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
LLVM_DEBUG(printPlans(dbgs()));
return;
@@ -5442,8 +5442,9 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM->collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, *CM);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ *CM);
LLVM_DEBUG(printPlans(dbgs()));
}
@@ -5591,9 +5592,11 @@ getRecordedExecutionFrequency(const VPBasicBlock *VPBB) {
}
#endif
-InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
- VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, *CM, Config,
+InstructionCost
+LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
+ VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const {
+ VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
LLVM_DEBUG(dbgs() << "Precomputed costs for VF " << VF << ": " << Cost
@@ -5734,7 +5737,7 @@ LoopVectorizationPlanner::computeBestVF() {
}
InstructionCost Cost =
- cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
+ cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, *CM);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
@@ -6380,7 +6383,8 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
}
#endif
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
+ LoopVectorizationCostModel &EnabledCM) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6413,8 +6417,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
if (IsInnerLoop) {
RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
- assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
- "execution frequencies do not match the loop's block frequencies");
+ assert(
+ verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, EnabledCM) &&
+ "execution frequencies do not match the loop's block frequencies");
}
// Create recipes for header phis. For outer loops, reductions, recurrences
@@ -6436,8 +6441,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6473,7 +6478,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM->foldTailByMasking())
+ if (EnabledCM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6481,16 +6486,17 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
return VPlan0;
}
-void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF) {
+void LoopVectorizationPlanner::buildVPlans(
+ VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- auto Plan =
- tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()), SubRange);
+ auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
+ SubRange, EnabledCM);
VF = SubRange.End;
if (!Plan)
@@ -6503,7 +6509,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (CM->foldTailWithEVL()) {
+ if (EnabledCM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6513,7 +6519,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = CM->getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6524,8 +6530,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
}
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
- VFRange &Range) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
+ VPlanPtr Plan, VFRange &Range, LoopVectorizationCostModel &EnabledCM) {
// For outer loops, the plan only needs basic recipe conversion and induction
// live-out optimization; the full inner-loop recipe building below does not
@@ -6551,8 +6557,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
- [this](ElementCount VF) {
- return !CM->requiresScalarEpilogue(VF.isVector());
+ [&EnabledCM](ElementCount VF) {
+ return !EnabledCM.requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6570,9 +6576,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(CM.get(), VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&EnabledCM, VF);
- TailFoldingStyle Style = CM->getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6598,10 +6604,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Range, add it to the set of groups to be later applied to the VPlan and add
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
- for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
- auto ApplyIG = [IG, this](ElementCount VF) -> bool {
+ for (InterleaveGroup<Instruction> *IG :
+ EnabledCM.InterleaveInfo.getInterleaveGroups()) {
+ auto ApplyIG = [IG, &EnabledCM](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- CM->getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM.getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6618,12 +6625,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, EnabledCM, Builder);
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6730,7 +6737,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM->isEpilogueAllowed());
+ InterleaveGroups, EnabledCM.isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6747,7 +6754,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (CM->maskPartialAliasing())
+ if (EnabledCM.maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
>From 8f46982a48074d8dc68099ae1030a1852f70594d Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 23 Sep 2026 13:13:45 +0100
Subject: [PATCH 2/2] add explaining comments for the new param
---
.../Vectorize/LoopVectorizationPlanner.h | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 3d2ded9ad0381..91daf9cadd8cf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -910,7 +910,9 @@ class LoopVectorizationPlanner {
/// A builder used to construct the current plan.
VPBuilder Builder;
- /// Computes the cost of \p Plan for vectorization factor \p VF.
+ /// Computes the cost of \p Plan for vectorization factor \p VF, using
+ /// \p EnabledCM as the cost model (the main loop's cost model, or a
+ /// separate one when costing a tail-folded epilogue).
///
/// The current implementation requires access to the
/// LoopVectorizationLegality to handle inductions and reductions, which is
@@ -1047,7 +1049,9 @@ class LoopVectorizationPlanner {
private:
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
- /// be built.
+ /// be built. \p EnabledCM is the cost model to use for decisions made while
+ /// building the plan (the main loop's cost model, or a separate one when
+ /// building a tail-folded epilogue's plan).
VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
/// Build a VPlan using VPRecipes according to the information gathered by
@@ -1057,13 +1061,17 @@ class LoopVectorizationPlanner {
/// can be built for the input range, set the largest included VF to the
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
- /// original scalar loop.
+ /// original scalar loop. \p EnabledCM is the cost model used for legality
+ /// and widening decisions while building the plan (the main loop's cost
+ /// model, or a separate one when building a tail-folded epilogue's plan).
VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
LoopVectorizationCostModel &EnabledCM);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
- /// when it checked if it is legal to vectorize the loop.
+ /// when it checked if it is legal to vectorize the loop. \p EnabledCM is
+ /// the cost model to build these VPlans (the main loop's cost model, or
+ /// a separate one when building a tail-folded epilogue's plans).
void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
LoopVectorizationCostModel &EnabledCM);
More information about the llvm-commits
mailing list