[llvm] [LV] Return best plan directly from selectBestEpiloguePlan (NFC). (PR #190608)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 06:57:23 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Change selectEpilogueVectorizationFactor to directly select the best VPlan and return a clone of it with the VF set to the selected VF.
This this replaces a getPlanFor outside of LVP and makes the interface more direct and more in line with computeBestVF after https://github.com/llvm/llvm-project/pull/190385.
---
Full diff: https://github.com/llvm/llvm-project/pull/190608.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h (+6-6)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+42-32)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ed00c3f6b2484..b0cf7823ce9f8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -611,12 +611,12 @@ class LoopVectorizationPlanner {
getDecisionAndClampRange(const std::function<bool(ElementCount)> &Predicate,
VFRange &Range);
- /// \return The most profitable vectorization factor and the cost of that VF
- /// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if
- /// epilogue vectorization is not supported for the loop.
- VectorizationFactor selectEpilogueVectorizationFactor(VPlan &MainPlan,
- ElementCount MainLoopVF,
- unsigned IC);
+ /// \return A VPlan for the most profitable epilogue vectorization, with its
+ /// VF narrowed to the chosen factor. The returned plan is a duplicate.
+ /// Returns nullptr if epilogue vectorization is not supported or not
+ /// profitable for the loop.
+ std::unique_ptr<VPlan>
+ selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45d0a5cac2688..79eb7db377d07 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4337,18 +4337,17 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
return estimateElementCount(VF * IC, VScaleForTuning) >= MinVFThreshold;
}
-VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
+std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
- VectorizationFactor Result = VectorizationFactor::Disabled();
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
- return Result;
+ return nullptr;
}
if (!CM.isScalarEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
"epilogue is allowed.\n");
- return Result;
+ return nullptr;
}
// Not really a cost consideration, but check for unsupported cases here to
@@ -4356,30 +4355,33 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
if (!isCandidateForEpilogueVectorization(MainPlan)) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
"is not a supported candidate.\n");
- return Result;
+ return nullptr;
}
if (EpilogueVectorizationForceVF > 1) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF);
- if (hasPlanWithVF(ForcedEC))
- return {ForcedEC, 0, 0};
+ if (hasPlanWithVF(ForcedEC)) {
+ std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC).duplicate());
+ Clone->setVF(ForcedEC);
+ return Clone;
+ }
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
"viable.\n");
- return Result;
+ return nullptr;
}
if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
LLVM_DEBUG(
dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
- return Result;
+ return nullptr;
}
if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
"this loop\n");
- return Result;
+ return nullptr;
}
// Check if a plan's vector loop processes fewer iterations than VF (e.g. when
@@ -4432,7 +4434,7 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
// No iterations left to process in the epilogue.
if (RemainingIterations->isZero())
- return Result;
+ return nullptr;
if (MainLoopVF.isFixed()) {
MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
@@ -4447,13 +4449,15 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
return SE.isKnownPredicate(CmpInst::ICMP_UGT, VF, RemIter);
};
+ VectorizationFactor Result = VectorizationFactor::Disabled();
+ VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
if (!hasPlanWithVF(NextVF.Width))
continue;
- ElementCount EffectiveVF =
- GetEffectiveVF(getPlanFor(NextVF.Width), NextVF.Width);
+ VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
// vectors) or > the VF of the main loop (fixed vectors).
if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
@@ -4483,14 +4487,20 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
if (Result.Width.isScalar() ||
isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
- /*IsEpilogue*/ true))
+ /*IsEpilogue*/ true)) {
Result = NextVF;
+ BestPlan = &CurrentPlan;
+ }
}
- if (Result != VectorizationFactor::Disabled())
- LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
- << Result.Width << "\n");
- return Result;
+ if (!BestPlan)
+ return nullptr;
+
+ LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
+ << Result.Width << "\n");
+ std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
+ Clone->setVF(Result.Width);
+ return Clone;
}
std::pair<unsigned, unsigned>
@@ -9603,45 +9613,45 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
- VectorizationFactor EpilogueVF =
- LVP.selectEpilogueVectorizationFactor(BestPlan, VF.Width, IC);
+ std::unique_ptr<VPlan> EpiPlan =
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
- if (EpilogueVF.Width.isVector()) {
- std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
+ if (EpiPlan) {
+ VPlan &BestEpiPlan = *EpiPlan;
+ VPlan &BestMainPlan = BestPlan;
+ ElementCount EpilogueVF = *BestEpiPlan.vectorFactors().begin();
// The first pass vectorizes the main loop and creates a scalar epilogue
// to be vectorized by executing the plan (potentially with a different
// factor) again shortly afterwards.
- VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
SmallVector<VPInstruction *> ResumeValues =
- preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
- EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
- BestEpiPlan);
+ preparePlanForMainVectorLoop(BestMainPlan, BestEpiPlan);
+ EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF, 1, BestEpiPlan);
// Add minimum iteration check for the epilogue plan, followed by runtime
// checks for the main plan.
- LVP.addMinimumIterationCheck(*BestMainPlan, EPI.EpilogueVF, EPI.EpilogueUF,
+ LVP.addMinimumIterationCheck(BestMainPlan, EPI.EpilogueVF, EPI.EpilogueUF,
ElementCount::getFixed(0));
- LVP.attachRuntimeChecks(*BestMainPlan, Checks, HasBranchWeights);
+ LVP.attachRuntimeChecks(BestMainPlan, Checks, HasBranchWeights);
VPlanTransforms::addIterationCountCheckBlock(
- *BestMainPlan, EPI.MainLoopVF, EPI.MainLoopUF,
+ BestMainPlan, EPI.MainLoopVF, EPI.MainLoopUF,
CM.requiresScalarEpilogue(EPI.MainLoopVF.isVector()), L,
HasBranchWeights ? MinItersBypassWeights : nullptr,
L->getLoopPredecessor()->getTerminator()->getDebugLoc(), PSE);
EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
- Checks, *BestMainPlan);
+ Checks, BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
- EPI.MainLoopVF, EPI.MainLoopUF, *BestMainPlan, MainILV, DT,
+ EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
// Derive EPI fields from VPlan-generated IR.
BasicBlock *EntryBB =
- cast<VPIRBasicBlock>(BestMainPlan->getEntry())->getIRBasicBlock();
+ cast<VPIRBasicBlock>(BestMainPlan.getEntry())->getIRBasicBlock();
EntryBB->setName("iter.check");
EPI.EpilogueIterationCountCheck = EntryBB;
// The check chain is: Entry -> [SCEV] -> [Mem] -> MainCheck -> VecPH.
``````````
</details>
https://github.com/llvm/llvm-project/pull/190608
More information about the llvm-commits
mailing list