[llvm] [LV] Return best VPlan together with VF from computeBestVF (NFC). (PR #190385)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 3 12:02:46 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/190385
computeBestVF iterates over all VPlans and picks the VF of the most profitable VPlan. This VPlan is later needed for execution and additional checks. Instead of retrieving it multiple times later, just directly return it from computeBestVF.
This removes some redundant lookups.
>From 678adbadd392d2101b9af1ae82fabe39ec77b9f3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 31 Mar 2026 20:47:30 +0100
Subject: [PATCH] [LV] Return best VPlan together with VF from computeBestVF
(NFC).
computeBestVF iterates over all VPlans and picks the VF of the most
profitable VPlan. This VPlan is later needed for execution and
additional checks. Instead of retrieving it multiple times later, just
directly return it from computeBestVF.
This removes some redundant lookups.
---
.../Vectorize/LoopVectorizationPlanner.h | 14 +++--
.../Transforms/Vectorize/LoopVectorize.cpp | 62 ++++++++++---------
2 files changed, 41 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bab9efee37533..ed00c3f6b2484 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -561,9 +561,10 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
- /// Compute and return the most profitable vectorization factor. Also collect
- /// all profitable VFs in ProfitableVFs.
- VectorizationFactor computeBestVF();
+ /// Compute and return the most profitable vectorization factor and the
+ /// corresponding best VPlan. Also collect all profitable VFs in
+ /// ProfitableVFs.
+ std::pair<VectorizationFactor, VPlan *> computeBestVF();
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -613,7 +614,8 @@ class LoopVectorizationPlanner {
/// \return The most profitable vectorization factor and the cost of that VF
/// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if
/// epilogue vectorization is not supported for the loop.
- VectorizationFactor selectEpilogueVectorizationFactor(ElementCount MainLoopVF,
+ VectorizationFactor selectEpilogueVectorizationFactor(VPlan &MainPlan,
+ ElementCount MainLoopVF,
unsigned IC);
/// Emit remarks for recipes with invalid costs in the available VPlans.
@@ -703,8 +705,8 @@ class LoopVectorizationPlanner {
bool IsEpilogue = false) const;
/// Determines if we have the infrastructure to vectorize the loop and its
- /// epilogue, assuming the main loop is vectorized by \p VF.
- bool isCandidateForEpilogueVectorization(const ElementCount VF) const;
+ /// epilogue, assuming the main loop is vectorized by \p MainPlan.
+ bool isCandidateForEpilogueVectorization(VPlan &MainPlan) const;
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5ddb47a22e84f..43986def67f4a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4278,7 +4278,7 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
}
bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
- ElementCount VF) const {
+ VPlan &MainPlan) const {
// Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
// reductions need special handling and are currently unsupported.
if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
@@ -4292,7 +4292,7 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
// FindLast reductions and inductions without underlying PHI require special
// handling and are currently not supported for epilogue vectorization.
- if (hasUnsupportedHeaderPhiRecipe(getPlanFor(VF)))
+ if (hasUnsupportedHeaderPhiRecipe(MainPlan))
return false;
// Phis with uses outside of the loop require special handling and are
@@ -4338,7 +4338,7 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
}
VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
- ElementCount MainLoopVF, unsigned IC) {
+ VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
VectorizationFactor Result = VectorizationFactor::Disabled();
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
@@ -4353,7 +4353,7 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
// Not really a cost consideration, but check for unsupported cases here to
// simplify the logic.
- if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
+ if (!isCandidateForEpilogueVectorization(MainPlan)) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
"is not a supported candidate.\n");
return Result;
@@ -4398,7 +4398,6 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
// Check if the main loop processes fewer than MainLoopVF elements per
// iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
// as needed.
- VPlan &MainPlan = getPlanFor(MainLoopVF);
MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
// If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
@@ -7148,13 +7147,14 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
}
#endif
-VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
+std::pair<VectorizationFactor, VPlan *>
+LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
- return VectorizationFactor::Disabled();
+ return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
- return {*FirstPlan.vectorFactors().begin(), 0, 0};
+ return {{*FirstPlan.vectorFactors().begin(), 0, 0}, &FirstPlan};
LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
<< (CM.CostKind == TTI::TCK_RecipThroughput
@@ -7167,7 +7167,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
: "Unknown\n"));
ElementCount ScalarVF = ElementCount::getFixed(1);
- assert(hasPlanWithVF(ScalarVF) &&
+ assert(FirstPlan.hasVF(ScalarVF) &&
"More than a single plan/VF w/o any plan having scalar VF");
// TODO: Compute scalar cost using VPlan-based cost model.
@@ -7184,6 +7184,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
BestFactor.Cost = InstructionCost::getMax();
}
+ VPlan *PlanForBestVF = &FirstPlan;
+
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
P->vectorFactors().end());
@@ -7219,8 +7221,10 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail()))
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
+ }
// If profitable add it to ProfitableVF list.
if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
@@ -7228,13 +7232,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
}
}
+ VPlan &BestPlan = *PlanForBestVF;
+
#ifndef NDEBUG
// Select the optimal vectorization factor according to the legacy cost-model.
// This is now only used to verify the decisions by the new VPlan-based
// cost-model and will be retired once the VPlan-based cost-model is
// stabilized.
VectorizationFactor LegacyVF = selectVectorizationFactor();
- VPlan &BestPlan = getPlanFor(BestFactor.Width);
// Pre-compute the cost and use it to check if BestPlan contains any
// simplifications not accounted for in the legacy cost model. If that's the
@@ -7258,20 +7263,20 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
!cast<VPWidenMemoryRecipe>(&R)->isConsecutive();
});
});
- assert(
- (BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
- !Legal->getLAI()->getSymbolicStrides().empty() || UsesEVLGatherScatter ||
- planContainsAdditionalSimplifications(
- getPlanFor(BestFactor.Width), CostCtx, OrigLoop, BestFactor.Width) ||
- planContainsAdditionalSimplifications(
- getPlanFor(LegacyVF.Width), CostCtx, OrigLoop, LegacyVF.Width)) &&
- " VPlan cost model and legacy cost model disagreed");
+ assert((BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
+ !Legal->getLAI()->getSymbolicStrides().empty() ||
+ UsesEVLGatherScatter ||
+ planContainsAdditionalSimplifications(BestPlan, CostCtx, OrigLoop,
+ BestFactor.Width) ||
+ planContainsAdditionalSimplifications(
+ getPlanFor(LegacyVF.Width), CostCtx, OrigLoop, LegacyVF.Width)) &&
+ " VPlan cost model and legacy cost model disagreed");
assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
"when vectorizing, the scalar cost must be computed.");
#endif
LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
- return BestFactor;
+ return {BestFactor, &BestPlan};
}
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
@@ -9423,7 +9428,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- VectorizationFactor VF = LVP.computeBestVF();
+ auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
if (ORE->allowExtraAnalysis(LV_NAME))
@@ -9432,7 +9437,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, CM.CostKind);
if (LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
- IC = LVP.selectInterleaveCount(LVP.getPlanFor(VF.Width), VF.Width, VF.Cost);
+ IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
unsigned SelectedIC = std::max(IC, UserIC);
// Optimistically generate runtime checks if they are needed. Drop them if
@@ -9455,12 +9460,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- VPCostContext CostCtx(CM.TTI, *CM.TLI, LVP.getPlanFor(VF.Width), CM,
- CM.CostKind, CM.PSE, L);
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, *BestPlanPtr, CM, CM.CostKind,
+ CM.PSE, L);
if (!ForceVectorization &&
- !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
- LVP.getPlanFor(VF.Width), SEL,
- CM.getVScaleForTuning())) {
+ !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
+ SEL, CM.getVScaleForTuning())) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
@@ -9597,10 +9601,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// If we decided that it is *legal* to interleave or vectorize the loop, then
// do it.
- VPlan &BestPlan = LVP.getPlanFor(VF.Width);
+ VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
VectorizationFactor EpilogueVF =
- LVP.selectEpilogueVectorizationFactor(VF.Width, IC);
+ LVP.selectEpilogueVectorizationFactor(BestPlan, VF.Width, IC);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpilogueVF.Width.isVector()) {
More information about the llvm-commits
mailing list