[llvm] [LV] codegen for tail-folded epilogue loop (PR #208764)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 24 06:01:10 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/208764
>From c1e2aac269d66dbf7c2baa16aa42c36450609d82 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 10 Jul 2026 16:06:34 +0100
Subject: [PATCH 1/3] [LV][EpilogueTailFolding] hack patch to start by codegen
---
.../Vectorize/LoopVectorizationPlanner.h | 24 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 186 +++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 25 ++-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +
.../Vectorize/VPlanConstruction.cpp | 18 +-
.../AArch64/fold-epilogue-tail.ll | 63 ++++++
.../LoopVectorize/fold-epilogue-tail.ll | 11 +-
7 files changed, 269 insertions(+), 67 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 9a06f36584ed0..f4f4b813d4e34 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -922,16 +922,17 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
- void plan(ElementCount UserVF, unsigned UserIC);
+ void plan(ElementCount UserVF, unsigned UserIC, bool IsEpilogueTFEnabled);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF) const;
+ VPlan &getPlanFor(ElementCount VF, bool TF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
- std::pair<VectorizationFactor, VPlan *> computeBestVF();
+ std::pair<VectorizationFactor, VPlan *>
+ computeBestVF(bool IsEpilogueTFEnabled);
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -957,6 +958,7 @@ class LoopVectorizationPlanner {
DenseMap<const SCEV *, Value *>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
+ bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind =
EpilogueVectorizationKind::None);
@@ -966,10 +968,7 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF) const {
- return any_of(VPlans,
- [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
- }
+ bool hasPlanWithVF(ElementCount VF, bool TF) const;
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
@@ -982,8 +981,10 @@ class LoopVectorizationPlanner {
/// VF narrowed to the chosen factor. The returned plan is a duplicate.
/// Returns nullptr if epilogue vectorization is not supported or not
/// profitable for the loop.
- std::unique_ptr<VPlan>
- selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC);
+ std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
+ ElementCount MainLoopVF,
+ unsigned IC,
+ bool IsEpilogueTFEnabled);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
@@ -1020,7 +1021,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1();
+ VPlanPtr tryToBuildVPlan1(bool IsEpilogueTFEnabled);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1035,7 +1036,8 @@ class LoopVectorizationPlanner {
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ bool IsEpilogueTFEnabled);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5f5767f836948..7b167d297f97f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3145,6 +3145,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
+ if (!Plan->isCompatibleWithTF(CM.foldTailByMasking()))
+ continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3446,7 +3448,8 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
}
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
- VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
+ VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
+ bool IsEpilogueTFEnabled) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
@@ -3486,9 +3489,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
- if (hasPlanWithVF(EpilogueVectorizationForceVF)) {
+ if (hasPlanWithVF(EpilogueVectorizationForceVF, CM.foldTailByMasking())) {
std::unique_ptr<VPlan> Clone(
- getPlanFor(EpilogueVectorizationForceVF).duplicate());
+ getPlanFor(EpilogueVectorizationForceVF, CM.foldTailByMasking())
+ .duplicate());
Clone->setVF(EpilogueVectorizationForceVF);
return Clone;
}
@@ -3579,10 +3583,12 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width))
+ if (!hasPlanWithVF(NextVF.Width,
+ CM.foldTailByMasking() || IsEpilogueTFEnabled))
continue;
- VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ VPlan &CurrentPlan =
+ getPlanFor(NextVF.Width, CM.foldTailByMasking() || IsEpilogueTFEnabled);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
// the VF of the main loop.
@@ -5480,7 +5486,8 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
}
-void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
+void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
+ bool IsEpilogueTFEnabled) {
CM.collectValuesToIgnore();
Config.collectElementTypesForWidening(&CM.ValuesToIgnore);
@@ -5495,7 +5502,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ false);
if (!VPlan1)
return;
@@ -5504,7 +5511,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF);
+ buildVPlans(*VPlan1, VF, VF, /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5547,9 +5554,10 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ false);
}
- buildVPlans(*VPlan1, UserVF, UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
if (!VPlans.empty() && VPlans.back()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
@@ -5580,10 +5588,27 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM.collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
-
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
+
+ // Build tail-folded vplans when IsEpilogueTFEnabled is enabled:
+ if (IsEpilogueTFEnabled) {
+ auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
+ if (!TFVPlan1)
+ return;
+ buildVPlans(*TFVPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ true);
+ buildVPlans(*TFVPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ true);
+ LLVM_DEBUG(dbgs() << "LV: Tail-folded vplans:\n");
+ for (auto &vplan : VPlans) {
+ if (vplan->isCompatibleWithTF(true))
+ LLVM_DEBUG(vplan->dump());
+ }
+ }
}
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
@@ -5785,7 +5810,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
}
std::pair<VectorizationFactor, VPlan *>
-LoopVectorizationPlanner::computeBestVF() {
+LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
@@ -5795,13 +5820,14 @@ LoopVectorizationPlanner::computeBestVF() {
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
+ assert((FirstPlan.hasScalarVFOnly() ||
+ hasPlanWithVF(UserVF, CM.foldTailByMasking()) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF()) {
+ if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) && hasForcedEpilogueVF()) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
assert(VPlans[0]->getSingleVF() == EpilogueVectorizationForceVF &&
"expected first plan to be for the forced epilogue VF");
@@ -5875,9 +5901,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
- BestFactor = CurrentFactor;
- PlanForBestVF = P.get();
+ if (P->isCompatibleWithTF(CM.foldTailByMasking())) {
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
+ BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
+ }
}
// If profitable add it to ProfitableVF list.
@@ -5897,7 +5925,7 @@ LoopVectorizationPlanner::computeBestVF() {
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
- InnerLoopVectorizer &ILV, DominatorTree *DT,
+ InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind) {
assert(BestVPlan.hasVF(BestVF) &&
"Trying to execute plan with unsupported VF");
@@ -6490,7 +6518,7 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6569,7 +6597,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM.foldTailByMasking())
+ if (CM.foldTailByMasking() || IsEpilogueTFEnabled)
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6577,7 +6605,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
}
void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF) {
+ ElementCount MaxVF,
+ bool IsEpilogueTFEnabled) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
@@ -6609,6 +6638,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
VPlans.push_back(std::move(P));
TailFoldingStyle Style = CM.getTailFoldingStyle();
+ if (IsEpilogueTFEnabled)
+ Style = TailFoldingStyle::Data;
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -7184,7 +7215,8 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
/// otherwise CM_EpilogueAllowed.
static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
- OptimizationRemarkEmitter *ORE) {
+ OptimizationRemarkEmitter *ORE,
+ LoopVectorizationLegality &LVL) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -7206,6 +7238,13 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (LVL.hasUncountableEarlyExit()) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
+ " of loop has early exit\n"
+ "LV: Fall back to a normal epilogue\n");
+ return CM_EpilogueAllowed;
+ }
+
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
@@ -7746,6 +7785,8 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
for (auto [ResumeV, HeaderPhi] :
zip(ResumeValues, BestEpiPlan.getScalarHeader()->phis())) {
auto *HeaderPhiR = cast<VPIRPhi>(&HeaderPhi);
+ if (!isa<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH)))
+ continue;
auto *EpiResumePhi =
cast<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH));
if (EpiResumePhi->getBasicBlockIndex(BypassBlock) == -1)
@@ -7762,15 +7803,15 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
/// and runtime checks of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
+static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
- DominatorTree *DT,
+ DominatorTree *DT, LoopInfo *LI,
GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove,
- ArrayRef<VPInstruction *> ResumeValues) {
+ ArrayRef<VPInstruction *> ResumeValues,
+ bool IsEpilogueTFEnabled) {
BasicBlock *VecEpilogueIterationCountCheck =
cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
-
BasicBlock *VecEpiloguePreHeader =
cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator())
->getSuccessor(1);
@@ -7794,7 +7835,11 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
BasicBlock *ScalarPH =
cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
- RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
+ // With a tail-folded epilogue there is no scalar remainder to bail
+ // to, even a trip count too small for the epilogue VF is handled safely by
+ // the masked epilogue vector loop, so skip straight to its preheader.
+ RedirectEdge(EPI.EpilogueIterationCountCheck,
+ IsEpilogueTFEnabled ? VecEpiloguePreHeader : ScalarPH);
// Adjust the terminators of runtime check blocks and phis using them.
BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
@@ -7825,11 +7870,20 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
return EPI.EpilogueIterationCountCheck == IncB;
}))
continue;
- for (BasicBlock *BB :
- {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
+ for (BasicBlock *BB : {SCEVCheckBlock, MemCheckBlock}) {
if (BB)
Phi->removeIncomingValue(BB);
}
+ // When the epilogue is tail-folded, EpilogueIterationCountCheck
+ // (iter.check) is redirected to branch straight into the vector epilogue
+ // preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
+ // genuine predecessor and its incoming value must be kept rather than
+ // stripped.
+ // TODO: revisit for reduction phis, whose resume value on this bypass
+ // edge may need dedicated handling rather than reusing the value already
+ // present here.
+ if (!IsEpilogueTFEnabled)
+ Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
}
auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
@@ -7847,6 +7901,46 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
for (PHINode &Phi : make_early_inc_range(VecEpiloguePreHeader->phis()))
if (Phi.use_empty())
Phi.eraseFromParent();
+
+ if (IsEpilogueTFEnabled) {
+ // The epilogue vector loop is tail-folded, so it can safely handle
+ // any remaining trip count, including zero, via masking.
+ // vec.epilog.iter.check's own min-iters check was therefore built with a
+ // compile-time-known-false condition (see
+ // addMinimumVectorEpilogueIterationCheck) that never needs to bail out to
+ // a scalar remainder. Fold it into an unconditional branch into the
+ // vector epilogue preheader.
+ auto *Br =
+ cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator());
+ [[maybe_unused]] auto *CondC = dyn_cast<ConstantInt>(Br->getCondition());
+ assert(CondC && CondC->isZero() &&
+ "expected vec.epilog.iter.check's branch condition to be a "
+ "compile-time false constant when the epilogue is tail-folded");
+ BasicBlock *DeadSucc = Br->getSuccessor(0);
+ UncondBrInst::Create(VecEpiloguePreHeader, Br->getIterator());
+ Br->eraseFromParent();
+ DTU.applyUpdates(
+ {{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
+ }
+
+ if (IsEpilogueTFEnabled && !SCEVCheckBlock && !MemCheckBlock) {
+ // No runtime check still needs a scalar fallback, and every trip-count
+ // bypass edge above has been redirected away from the scalar preheader:
+ // the scalar loop is now entirely unreachable. Delete it outright, along
+ // with its LoopInfo entry, instead of leaving it behind as dead code.
+ // This must run last, since fixScalarResumeValuesFromBypass (above) and
+ // the DT/function verification in processLoop (below) still need `L` and
+ // ScalarPH to be valid up to this point.
+ assert(pred_empty(ScalarPH) &&
+ "scalar preheader should have no predecessors left");
+ SmallVector<BasicBlock *> Blocks(L->block_begin(),
+ L->block_end());
+ Blocks.push_back(ScalarPH);
+ LI->erase(L);
+ for (auto *BB : Blocks)
+ LI->removeBlock(BB);
+ DeleteDeadBlocks(Blocks, &DTU);
+ }
}
bool LoopVectorizePass::processLoop(Loop *L) {
@@ -8037,15 +8131,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Hints, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE);
+ getEpilogueTailLowering(CM, L, ORE, LVL);
+ bool IsEpilogueTFEnabled = false;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
- reportVectorizationInfo(
- "The epilogue-tail-folding policy prefer-fold-tail is not supported "
- "yet, fall back to a normal epilogue",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
+ IsEpilogueTFEnabled = true;
}
// Get user vectorization factor and interleave count.
@@ -8059,8 +8151,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
UserIC = 1;
// Plan how to best vectorize.
- LVP.plan(UserVF, UserIC);
- auto [VF, BestPlanPtr] = LVP.computeBestVF();
+ LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
+ auto [VF, BestPlanPtr] = LVP.computeBestVF(IsEpilogueTFEnabled);
unsigned IC = 1;
// For VPlan build stress testing of outer loops, bail after plan
@@ -8076,7 +8168,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
CM.maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8137,7 +8229,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) &&
+ UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
@@ -8263,7 +8356,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
std::unique_ptr<VPlan> EpiPlan =
- LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC);
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, IsEpilogueTFEnabled);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
@@ -8296,6 +8389,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
+ /*IsEpilogueTFEnabled*/ false,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
@@ -8327,9 +8421,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LVP.attachRuntimeChecks(BestEpiPlan, Checks, HasBranchWeights);
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
+ IsEpilogueTFEnabled,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
- ResumeValues);
+ connectEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L, EPI, DT, LI, Checks,
+ InstsToMove, ResumeValues, IsEpilogueTFEnabled);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
@@ -8341,7 +8436,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (!IsInnerLoop)
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
<< "\"\n");
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT);
+ LVP.executePlan(VF.Width, IC, BestPlan, LB, DT,
+ /*IsEpilogueTFEnabled*/ false);
++LoopsVectorized;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fe4c15e34038b..343863dea3ab1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1322,6 +1322,13 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
+bool VPlan::isCompatibleWithTF(bool TF) {
+ auto *VLR = getVectorLoopRegion();
+ assert(VLR && "Vector loop region got eliminated\n");
+ bool HasHeaderMask = (VLR->getHeaderMask() != nullptr);
+ return HasHeaderMask == TF;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1691,19 +1698,29 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
assert(count_if(VPlans,
- [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
- 1 &&
+ [VF, TF](const VPlanPtr &Plan) {
+ LLVM_DEBUG(dbgs() << "LV: given VF: " << VF << " and TF: "
+ << TF << " equivalent vplan: ";
+ Plan->dump());
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ }) == 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF))
+ if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
+bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
+ return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ });
+}
+
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index edb24141f6073..74ac561b0353d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5195,6 +5195,15 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
+ /// Returns true if this VPlan's tail-folding matches \p TF.
+ /// A plan is tail-folded if it has a header mask, or if it has no scalar
+ /// tail while the TC is not evenly divisible by VF (in which case the tail
+ /// must be folded by the vector loop itself).
+ ///
+ /// This function is used to disambiguate VPlan lookup by VF when multiple
+ /// plans share the same VF but differ in tail-folding status.
+ bool isCompatibleWithTF(bool TF);
+
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 34fbf85cbad42..f34bdacd882b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1582,9 +1582,25 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep,
unsigned EpilogueLoopStep, ScalarEvolution &SE) {
// Add the minimum iteration check for the epilogue vector loop.
+ VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
+
+ if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
+ return;
+ }
+ // if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ // VPBasicBlock *EntryVPBB = cast<VPBasicBlock>(Plan.getEntry());
+ // // Successor 0 is the "taken" (scalar) edge, successor 1 is the vector
+ // // path — see BranchOnCond's execute() and removeBranchOnConst's
+ // // RemovedIdx convention. Drop the scalar edge directly, leaving Entry
+ // // with a single successor (which needs no BranchOnCond terminator at
+ // // all — VPlan codegen emits a plain unconditional `br` for that case).
+ // VPBlockUtils::disconnectBlocks(EntryVPBB, EntryVPBB->getSuccessors()[0]);
+ // return;
+ // }
+
VPValue *TC = Plan.getTripCount();
Value *TripCount = TC->getLiveInIRValue();
- VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
VPValue *VFxUF = Builder.createExpandSCEV(SE.getElementCount(
TripCount->getType(), (EpilogueVF * EpilogueUF), SCEV::FlagNUW));
VPValue *Count = Builder.createSub(TC, Plan.getOrAddLiveIn(VectorTripCount),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
new file mode 100644
index 0000000000000..1cbd4b225efd9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+
+
+define void @test_epilogue_tf(ptr %A, i64 %n) {
+; CHECK-LABEL: define void @test_epilogue_tf(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP0]], align 1
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 15
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 590907a5aff79..6bb1f7349413a 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,15 +1,14 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail\
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 \
+; RUN: opt -S -p loop-vectorize -debug -enable-epilogue-vectorization=false \
+; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize < %s 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: epilogue tail-folding is not supported yet
-; CHECK: remark: <unknown>:0:0: The epilogue-tail-folding policy prefer-fold-tail is not supported yet, fall back to a normal epilogue
+; CHECK: LV: epilogue tail-folding is enabled
;
entry:
br label %for.body
>From 25f4bbb7c52a264be4719641870d2e62a8da2ad6 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 21 Jul 2026 15:10:26 +0100
Subject: [PATCH 2/3] rebase and resolve review comments - limit the feature to
be enabled only for forced epilogue VF
---
.../Vectorize/LoopVectorizationPlanner.h | 10 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 72 ++++++++-----------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 25 ++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 9 ---
.../Vectorize/VPlanConstruction.cpp | 12 +---
.../AArch64/fold-epilogue-tail.ll | 19 +++--
.../LoopVectorize/fold-epilogue-tail.ll | 42 +++++++++--
7 files changed, 89 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f4f4b813d4e34..775e915ceaefc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -926,13 +926,12 @@ class LoopVectorizationPlanner {
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF, bool TF) const;
+ VPlan &getPlanFor(ElementCount VF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
- std::pair<VectorizationFactor, VPlan *>
- computeBestVF(bool IsEpilogueTFEnabled);
+ std::pair<VectorizationFactor, VPlan *> computeBestVF();
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -968,7 +967,10 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF, bool TF) const;
+ bool hasPlanWithVF(ElementCount VF) const {
+ return any_of(VPlans,
+ [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
+ }
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7b167d297f97f..d3d1f69175ea4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3145,8 +3145,6 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
- if (!Plan->isCompatibleWithTF(CM.foldTailByMasking()))
- continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3489,10 +3487,9 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
- if (hasPlanWithVF(EpilogueVectorizationForceVF, CM.foldTailByMasking())) {
+ if (hasPlanWithVF(EpilogueVectorizationForceVF)) {
std::unique_ptr<VPlan> Clone(
- getPlanFor(EpilogueVectorizationForceVF, CM.foldTailByMasking())
- .duplicate());
+ getPlanFor(EpilogueVectorizationForceVF).duplicate());
Clone->setVF(EpilogueVectorizationForceVF);
return Clone;
}
@@ -3583,12 +3580,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width,
- CM.foldTailByMasking() || IsEpilogueTFEnabled))
+ if (!hasPlanWithVF(NextVF.Width))
continue;
- VPlan &CurrentPlan =
- getPlanFor(NextVF.Width, CM.foldTailByMasking() || IsEpilogueTFEnabled);
+ VPlan &CurrentPlan = getPlanFor(NextVF.Width);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
// the VF of the main loop.
@@ -5554,8 +5549,13 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ false);
+ if (IsEpilogueTFEnabled) {
+ auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
+ buildVPlans(*TFVPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ true);
+ } else
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ false);
}
buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
if (!VPlans.empty() && VPlans.back()->getSingleVF() == UserVF) {
@@ -5593,22 +5593,6 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
/*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
-
- // Build tail-folded vplans when IsEpilogueTFEnabled is enabled:
- if (IsEpilogueTFEnabled) {
- auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
- if (!TFVPlan1)
- return;
- buildVPlans(*TFVPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
- /*IsEpilogueTFEnabled*/ true);
- buildVPlans(*TFVPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
- /*IsEpilogueTFEnabled*/ true);
- LLVM_DEBUG(dbgs() << "LV: Tail-folded vplans:\n");
- for (auto &vplan : VPlans) {
- if (vplan->isCompatibleWithTF(true))
- LLVM_DEBUG(vplan->dump());
- }
- }
}
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
@@ -5810,7 +5794,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
}
std::pair<VectorizationFactor, VPlan *>
-LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
+LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
@@ -5820,14 +5804,13 @@ LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() ||
- hasPlanWithVF(UserVF, CM.foldTailByMasking()) ||
+ assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) && hasForcedEpilogueVF()) {
+ if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF()) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
assert(VPlans[0]->getSingleVF() == EpilogueVectorizationForceVF &&
"expected first plan to be for the forced epilogue VF");
@@ -5901,11 +5884,9 @@ LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (P->isCompatibleWithTF(CM.foldTailByMasking())) {
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
- BestFactor = CurrentFactor;
- PlanForBestVF = P.get();
- }
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
+ BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
}
// If profitable add it to ProfitableVF list.
@@ -7216,7 +7197,8 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
OptimizationRemarkEmitter *ORE,
- LoopVectorizationLegality &LVL) {
+ LoopVectorizationLegality &LVL,
+ LoopVectorizeHints &Hints) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -7230,6 +7212,13 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (!hasForcedEpilogueVF() || !Hints.getWidth()) {
+ reportVectorizationInfo("For now, Epilogue tail-folding can't be "
+ "applied without forced epilogue/main loop VF\n",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
// If scalar epilogue is explicitly required, we can't apply TF.
if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
@@ -8131,7 +8120,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Hints, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE, LVL);
+ getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
bool IsEpilogueTFEnabled = false;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
@@ -8152,7 +8141,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
- auto [VF, BestPlanPtr] = LVP.computeBestVF(IsEpilogueTFEnabled);
+ auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
// For VPlan build stress testing of outer loops, bail after plan
@@ -8168,7 +8157,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
CM.maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8229,8 +8218,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) &&
- UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 343863dea3ab1..fe4c15e34038b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1322,13 +1322,6 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
-bool VPlan::isCompatibleWithTF(bool TF) {
- auto *VLR = getVectorLoopRegion();
- assert(VLR && "Vector loop region got eliminated\n");
- bool HasHeaderMask = (VLR->getHeaderMask() != nullptr);
- return HasHeaderMask == TF;
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1698,29 +1691,19 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
assert(count_if(VPlans,
- [VF, TF](const VPlanPtr &Plan) {
- LLVM_DEBUG(dbgs() << "LV: given VF: " << VF << " and TF: "
- << TF << " equivalent vplan: ";
- Plan->dump());
- return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
- }) == 1 &&
+ [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
+ 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
+ if (Plan->hasVF(VF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
-bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
- return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
- return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
- });
-}
-
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 74ac561b0353d..edb24141f6073 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5195,15 +5195,6 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
- /// Returns true if this VPlan's tail-folding matches \p TF.
- /// A plan is tail-folded if it has a header mask, or if it has no scalar
- /// tail while the TC is not evenly divisible by VF (in which case the tail
- /// must be folded by the vector loop itself).
- ///
- /// This function is used to disambiguate VPlan lookup by VF when multiple
- /// plans share the same VF but differ in tail-folding status.
- bool isCompatibleWithTF(bool TF);
-
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index f34bdacd882b2..5461b94fb6796 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1584,20 +1584,10 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
// Add the minimum iteration check for the epilogue vector loop.
VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
- if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ if (Plan.hasTailFolded()) {
Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
return;
}
- // if (Plan.isCompatibleWithTF(/*TF*/ true)) {
- // VPBasicBlock *EntryVPBB = cast<VPBasicBlock>(Plan.getEntry());
- // // Successor 0 is the "taken" (scalar) edge, successor 1 is the vector
- // // path — see BranchOnCond's execute() and removeBranchOnConst's
- // // RemovedIdx convention. Drop the scalar edge directly, leaving Entry
- // // with a single successor (which needs no BranchOnCond terminator at
- // // all — VPlan codegen emits a plain unconditional `br` for that case).
- // VPBlockUtils::disconnectBlocks(EntryVPBB, EntryVPBB->getSuccessors()[0]);
- // return;
- // }
VPValue *TC = Plan.getTripCount();
Value *TripCount = TC->getLiveInIRValue();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 1cbd4b225efd9..9a2785a10daf0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: define void @test_epilogue_tf(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
@@ -31,15 +32,15 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 15
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 7
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N_RND_UP]], 8
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP3]], align 1
-; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
+; CHECK-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -61,3 +62,9 @@ for.body:
exit:
ret void
}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 6bb1f7349413a..2407c5e28c892 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,13 +1,19 @@
; REQUIRES: asserts
; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail\
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s
+; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s
+
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
; RUN: opt -S -p loop-vectorize -debug -enable-epilogue-vectorization=false \
-; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize < %s 2>&1 \
-; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+; RUN: --disable-output -force-vector-width=16 -epilogue-vectorization-force-VF=8 -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
define void @test_epilogue_tf(ptr %A, i64 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is enabled
;
entry:
@@ -25,8 +31,30 @@ exit:
ret void
}
+define void @test_epilogue_tf_no_fv(ptr %A, i64 %n) {
+; CHECK-NO-FORCED-MAIN-VF-LABEL: Checking a loop in 'test_epilogue_tf_no_fv'
+; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced epilogue/main loop VF
+
+; CHECK-NO-FORCED-EPILOGUE-VF-LABEL: Checking a loop in 'test_epilogue_tf_no_fv'
+; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced epilogue/main loop VF
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
define void @epilogue_is_disabled(ptr %a, i64 %n) {
-; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG-LABEL: Checking a loop in 'epilogue_is_disabled'
; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
;
entry:
@@ -45,7 +73,7 @@ for.end:
}
define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
-; CHECK-LABEL: LV: Checking a loop in 'require_scalar_epilogue'
+; CHECK-LABEL: Checking a loop in 'require_scalar_epilogue'
; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
; CHECK-NEXT: LV: Fall back to a normal epilogue
;
@@ -74,7 +102,7 @@ exit.2:
}
define i32 @opt_for_size(ptr %p, i32 %n) optsize {
-; CHECK-LABEL: LV: Checking a loop in 'opt_for_size'
+; CHECK-LABEL: Checking a loop in 'opt_for_size'
; CHECK: LV: No epilogue to apply tail-folding for.
; CHECK-NEXT: LV: Fall back to a normal epilogue
;
>From f943002b203b8d6a1e65abc9ef35a230be1e0edf Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 23 Jul 2026 11:57:23 +0100
Subject: [PATCH 3/3] Enable correct costs for tail-folded epilogue. To achieve
that, I pass the epilogue CM to cost/vplan-related functions.
---
.../Vectorize/LoopVectorizationPlanner.h | 26 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 263 ++++++++++++------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 6 +
.../AArch64/fold-epilogue-tail.ll | 25 +-
4 files changed, 206 insertions(+), 114 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 775e915ceaefc..b28cc35454672 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -901,7 +901,8 @@ class LoopVectorizationPlanner {
///
/// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
/// been retired.
- InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
+ InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const;
/// Precompute costs for certain instructions using the legacy cost model. The
/// function is used to bring up the VPlan-based cost model to initially avoid
@@ -922,7 +923,14 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
- void plan(ElementCount UserVF, unsigned UserIC, bool IsEpilogueTFEnabled);
+ void plan(ElementCount UserVF, unsigned UserIC);
+
+ /// Build VPlans for the specified \p EpilogueUserVF and \p IC if they are
+ /// non-zero or all applicable candidate VFs otherwise. If vectorization and
+ /// tail-folding should be avoided up-front, no plans are generated.
+ bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC,
+ ElementCount EpilogueUserVF,
+ LoopVectorizationCostModel &EpilogueCM);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
@@ -957,7 +965,6 @@ class LoopVectorizationPlanner {
DenseMap<const SCEV *, Value *>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
- bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind =
EpilogueVectorizationKind::None);
@@ -983,10 +990,8 @@ class LoopVectorizationPlanner {
/// VF narrowed to the chosen factor. The returned plan is a duplicate.
/// Returns nullptr if epilogue vectorization is not supported or not
/// profitable for the loop.
- std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
- ElementCount MainLoopVF,
- unsigned IC,
- bool IsEpilogueTFEnabled);
+ std::unique_ptr<VPlan>
+ selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
@@ -1023,7 +1028,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1(bool IsEpilogueTFEnabled);
+ VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1033,13 +1038,14 @@ class LoopVectorizationPlanner {
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
/// original scalar loop.
- VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
+ VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
+ LoopVectorizationCostModel &EnabledCM);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
- bool IsEpilogueTFEnabled);
+ LoopVectorizationCostModel &EnabledCM);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d3d1f69175ea4..293f7a333d9a6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3446,8 +3446,7 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
}
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
- VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
- bool IsEpilogueTFEnabled) {
+ VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
@@ -3687,7 +3686,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (VF.isScalar())
LoopCost = CM.expectedCost(VF);
else
- LoopCost = cost(Plan, VF, &R);
+ LoopCost = cost(Plan, VF, &R, CM);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
// Loop body is free and there is no need for interleaving.
@@ -5481,8 +5480,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
}
-void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
- bool IsEpilogueTFEnabled) {
+void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM.collectValuesToIgnore();
Config.collectElementTypesForWidening(&CM.ValuesToIgnore);
@@ -5497,7 +5495,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ false);
+ auto VPlan1 = tryToBuildVPlan1(CM);
if (!VPlan1)
return;
@@ -5506,7 +5504,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF, /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, VF, VF, CM);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5545,24 +5543,19 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
CM.collectNonVectorizedAndSetWideningDecisions(UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, CM);
+
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- if (IsEpilogueTFEnabled) {
- auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
- buildVPlans(*TFVPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ true);
- } else
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, CM);
}
- buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
- if (!VPlans.empty() && VPlans.back()->getSingleVF() == UserVF) {
+ if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
if (UserVF.isScalar() ||
- cost(*VPlans.back(), UserVF, /*RU=*/nullptr).isValid()) {
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr, CM).isValid()) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
LLVM_DEBUG(printPlans(dbgs()));
return;
@@ -5588,13 +5581,73 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
CM.collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
- /*IsEpilogueTFEnabled*/ false);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
- /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, CM);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF, CM);
+
LLVM_DEBUG(printPlans(dbgs()));
}
+bool LoopVectorizationPlanner::planForEpilogueTF(
+ ElementCount UserVF, unsigned UserIC, ElementCount EpilogueUserVF,
+ LoopVectorizationCostModel &EpilogueCM) {
+ if (VPlans.empty())
+ return false;
+ if (!OrigLoop->isInnermost())
+ return false;
+
+ if (!EpilogueUserVF.isVector() ||
+ ElementCount::isKnownGE(EpilogueUserVF, UserVF))
+ return false;
+
+ EpilogueCM.ValuesToIgnore.insert_range(CM.ValuesToIgnore);
+ EpilogueCM.VecValuesToIgnore.insert_range(CM.VecValuesToIgnore);
+
+ FixedScalableVFPair MaxFactors =
+ EpilogueCM.computeMaxVF(EpilogueUserVF, UserIC);
+ if (!MaxFactors ||
+ !EpilogueCM.foldTailByMasking()) // Cases that should not to be vectorized
+ // nor tail-folded.
+ return false;
+
+ auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
+ if (!VPlan1)
+ return false;
+
+ // Invalidate interleave groups if all blocks of loop will be predicated.
+ if (EpilogueCM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ !useMaskedInterleavedAccesses(TTI)) {
+ LLVM_DEBUG(
+ dbgs() << "LV: [EpilogueTF] Invalidate all interleaved groups due to "
+ "fold-tail "
+ "by masking which requires masked-interleaved support.\n");
+ if (EpilogueCM.InterleaveInfo.invalidateGroups())
+ // Invalidating interleave groups also requires invalidating all decisions
+ // based on them, which includes widening decisions and uniform and scalar
+ // values.
+ EpilogueCM.invalidateCostModelingDecisions();
+ }
+
+ if (EpilogueCM.foldTailByMasking())
+ Legal->prepareToFoldTailByMasking();
+
+ // Collect the instructions (and their associated costs) that will be more
+ // profitable to scalarize.
+ EpilogueCM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+
+ assert(VPlans.size() == 2 &&
+ "For tail-folded epilogue, VPlans size is expected to be 2");
+ // remove last vplan which should be the epilogue plan to replace it by our
+ // tail-folded vplan:
+ assert(VPlans.back()->getSingleVF() == EpilogueUserVF &&
+ "For tail-folded epilogue, first vplan is expected to have "
+ "EpilogueUserVF");
+ VPlans.pop_back();
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, EpilogueCM);
+
+ cost(*VPlans.back(), EpilogueUserVF, /*RU=*/nullptr, EpilogueCM);
+ return true;
+}
+
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
LoopVectorizationCostModel &CM,
VFSelectionContext &Config)
@@ -5719,7 +5772,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
// accounted for here using the legacy cost model. However, some opcodes
// are excluded from these precomputed scalarization costs and are instead
// modeled later by the VPlan cost model (see UseVPlanCostModel below).
- for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
+ for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
if (CostCtx.skipCostComputation(ForcedScalar, VF.isVector()))
continue;
CostCtx.SkipCostComputation.insert(ForcedScalar);
@@ -5742,7 +5795,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return false;
}
};
- for (const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
+ for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
if (UseVPlanCostModel(Scalarized) ||
CostCtx.skipCostComputation(Scalarized, VF.isVector()))
continue;
@@ -5757,9 +5810,11 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return Cost;
}
-InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
- VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, CM, Config);
+InstructionCost
+LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
+ VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const {
+ VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
// Now compute and add the VPlan-based cost.
@@ -5812,11 +5867,11 @@ LoopVectorizationPlanner::computeBestVF() {
if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF()) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
- assert(VPlans[0]->getSingleVF() == EpilogueVectorizationForceVF &&
- "expected first plan to be for the forced epilogue VF");
- assert(VPlans[1]->getSingleVF() == UserVF &&
- "expected second plan to be for the forced UserVF");
- return {VectorizationFactor(UserVF, 0, 0), VPlans[1].get()};
+ assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
+ "expected second plan to be for the forced epilogue VF");
+ assert(VPlans[0]->getSingleVF() == UserVF &&
+ "expected first plan to be for the forced UserVF");
+ return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
}
LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
@@ -5881,7 +5936,7 @@ LoopVectorizationPlanner::computeBestVF() {
}
InstructionCost Cost =
- cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
+ cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, CM);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
@@ -5906,7 +5961,7 @@ LoopVectorizationPlanner::computeBestVF() {
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
- InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueTFEnabled,
+ InnerLoopVectorizer &ILV, DominatorTree *DT,
EpilogueVectorizationKind EpilogueVecKind) {
assert(BestVPlan.hasVF(BestVF) &&
"Trying to execute plan with unsupported VF");
@@ -6499,7 +6554,8 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
+ LoopVectorizationCostModel &EnabledCM) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6549,8 +6605,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6578,24 +6634,24 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM.foldTailByMasking() || IsEpilogueTFEnabled)
+ if (EnabledCM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
}
-void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF,
- bool IsEpilogueTFEnabled) {
+void LoopVectorizationPlanner::buildVPlans(
+ VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- auto Plan =
- tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()), SubRange);
+ auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
+ SubRange, EnabledCM);
VF = SubRange.End;
if (!Plan)
@@ -6608,7 +6664,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (CM.foldTailWithEVL()) {
+ if (EnabledCM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6618,9 +6674,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = CM.getTailFoldingStyle();
- if (IsEpilogueTFEnabled)
- Style = TailFoldingStyle::Data;
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6631,8 +6685,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
}
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
- VFRange &Range) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
+ VPlanPtr Plan, VFRange &Range, LoopVectorizationCostModel &EnabledCM) {
// For outer loops, the plan only needs basic recipe conversion and induction
// live-out optimization; the full inner-loop recipe building below does not
@@ -6658,8 +6712,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
- [this](ElementCount VF) {
- return !CM.requiresScalarEpilogue(VF.isVector());
+ [EnabledCM](ElementCount VF) {
+ return !EnabledCM.requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6677,9 +6731,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&EnabledCM, VF);
- TailFoldingStyle Style = CM.getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6705,10 +6759,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Range, add it to the set of groups to be later applied to the VPlan and add
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
- for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
- auto ApplyIG = [IG, this](ElementCount VF) -> bool {
+ for (InterleaveGroup<Instruction> *IG :
+ EnabledCM.InterleaveInfo.getInterleaveGroups()) {
+ auto ApplyIG = [IG, EnabledCM](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- CM.getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM.getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6725,7 +6780,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, EnabledCM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6736,7 +6791,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, CM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6836,7 +6891,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// range for better cost estimation.
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
- if (!CM.foldTailWithEVL()) {
+ if (!EnabledCM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
@@ -6847,7 +6902,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM.isEpilogueAllowed());
+ InterleaveGroups, EnabledCM.isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6864,7 +6919,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (CM.maskPartialAliasing())
+ if (EnabledCM.maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -7576,7 +7631,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
assert(all_of(IV->users(),
[](const VPUser *U) {
- if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
+ if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe,
+ VPWidenCanonicalIVRecipe>(U))
return true;
unsigned Opc = cast<VPInstruction>(U)->getOpcode();
return Instruction::isCast(Opc) || Opc == Instruction::Add;
@@ -7590,7 +7646,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto *Increment = vputils::findCanonicalIVIncrement(Plan);
assert(Increment && "Must have a canonical IV increment at this point");
IV->replaceUsesWithIf(Add, [Add, Increment](VPUser &U, unsigned) {
- return &U != Add && &U != Increment;
+ return &U != Add && &U != Increment && !isa<VPWidenCanonicalIVRecipe>(&U);
});
VPInstruction *OffsetIVInc =
VPBuilder::getToInsertAfter(Increment).createAdd(Increment, VPV);
@@ -7701,6 +7757,23 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
continue;
}
}
+ } else if (isa<VPActiveLaneMaskPHIRecipe>(&R)) {
+ // The active-lane-mask phi's entry value was computed assuming the
+ // epilogue vector loop starts at 0. Rebuild it using the resume value
+ // instead, so the mask reflects how many elements the main vector loop
+ // already processed.
+ VPBuilder EntryBuilder(Plan.getVectorPreheader());
+ Type *CanIVTy = VectorLoop->getCanonicalIVType();
+ VPValue *ALMMultiplier = Plan.getConstantInt(CanIVTy, 1);
+ auto *EntryIncrement = EntryBuilder.createOverflowingOp(
+ VPInstruction::CanonicalIVIncrementForPart, {VPV, &Plan.getVF()}, {},
+ R.getDebugLoc(), "index.part.next");
+ auto *EntryALM = EntryBuilder.createNaryOp(
+ VPInstruction::ActiveLaneMask,
+ {EntryIncrement, Plan.getTripCount(), ALMMultiplier}, R.getDebugLoc(),
+ "active.lane.mask.entry");
+ cast<VPHeaderPHIRecipe>(&R)->setStartValue(EntryALM);
+ continue;
} else {
// Retrieve the induction resume value via ResumeForEpilogue.
PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
@@ -7792,7 +7865,7 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
/// and runtime checks of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
+static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
DominatorTree *DT, LoopInfo *LI,
GeneratedRTChecks &Checks,
@@ -7866,8 +7939,7 @@ static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
// When the epilogue is tail-folded, EpilogueIterationCountCheck
// (iter.check) is redirected to branch straight into the vector epilogue
// preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
- // genuine predecessor and its incoming value must be kept rather than
- // stripped.
+ // predecessor and its incoming value must be kept rather than stripped.
// TODO: revisit for reduction phis, whose resume value on this bypass
// edge may need dedicated handling rather than reusing the value already
// present here.
@@ -7910,25 +7982,18 @@ static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
Br->eraseFromParent();
DTU.applyUpdates(
{{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
- }
- if (IsEpilogueTFEnabled && !SCEVCheckBlock && !MemCheckBlock) {
- // No runtime check still needs a scalar fallback, and every trip-count
- // bypass edge above has been redirected away from the scalar preheader:
- // the scalar loop is now entirely unreachable. Delete it outright, along
- // with its LoopInfo entry, instead of leaving it behind as dead code.
- // This must run last, since fixScalarResumeValuesFromBypass (above) and
- // the DT/function verification in processLoop (below) still need `L` and
- // ScalarPH to be valid up to this point.
- assert(pred_empty(ScalarPH) &&
- "scalar preheader should have no predecessors left");
- SmallVector<BasicBlock *> Blocks(L->block_begin(),
- L->block_end());
- Blocks.push_back(ScalarPH);
- LI->erase(L);
- for (auto *BB : Blocks)
- LI->removeBlock(BB);
- DeleteDeadBlocks(Blocks, &DTU);
+ if (!SCEVCheckBlock && !MemCheckBlock) {
+ // Delete the scalar loop as it's dead right
+ assert(pred_empty(ScalarPH) &&
+ "scalar preheader should have no predecessors left");
+ SmallVector<BasicBlock *> Blocks(L->block_begin(), L->block_end());
+ Blocks.push_back(ScalarPH);
+ LI->erase(L);
+ for (auto *BB : Blocks)
+ LI->removeBlock(BB);
+ DeleteDeadBlocks(Blocks, &DTU);
+ }
}
}
@@ -8122,11 +8187,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
bool IsEpilogueTFEnabled = false;
+ std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
+ std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
- // TODO: Apply tail-folding on the vectorized epilogue loop.
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
IsEpilogueTFEnabled = true;
+ TailFoldingCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
+ if (UseInterleaved && useMaskedInterleavedAccesses(*TTI))
+ TailFoldingCMIAI->analyzeInterleaving(
+ /*useMaskedInterleavedAccesses*/ true);
+ EpilogueTailFoldingCM.emplace(CM_EpilogueNotNeededFoldTail, L, PSE, LI,
+ &LVL, *TTI, TLI, AC, ORE, GetBFI, F, &Hints,
+ *TailFoldingCMIAI, Config);
}
// Get user vectorization factor and interleave count.
@@ -8140,7 +8213,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
UserIC = 1;
// Plan how to best vectorize.
- LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
+ LVP.plan(UserVF, UserIC);
+ if (IsEpilogueTFEnabled)
+ if (!LVP.planForEpilogueTF(UserVF, /*UserIC*/ 1,
+ EpilogueVectorizationForceVF,
+ *EpilogueTailFoldingCM)) {
+ // we can't apply epilogue TF:
+ EpilogueTailFoldingCM.reset();
+ IsEpilogueTFEnabled = false;
+ }
+
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
@@ -8344,7 +8426,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
std::unique_ptr<VPlan> EpiPlan =
- LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, IsEpilogueTFEnabled);
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
@@ -8377,7 +8459,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
- /*IsEpilogueTFEnabled*/ false,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
@@ -8409,10 +8490,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LVP.attachRuntimeChecks(BestEpiPlan, Checks, HasBranchWeights);
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
- IsEpilogueTFEnabled,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L, EPI, DT, LI, Checks,
- InstsToMove, ResumeValues, IsEpilogueTFEnabled);
+ connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
+ ResumeValues, IsEpilogueTFEnabled);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
@@ -8424,8 +8504,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (!IsInnerLoop)
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
<< "\"\n");
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT,
- /*IsEpilogueTFEnabled*/ false);
+ LVP.executePlan(VF.Width, IC, BestPlan, LB, DT);
++LoopsVectorized;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fe4c15e34038b..0d250a777ac9c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -751,6 +751,12 @@ VPRegionBlock *VPRegionBlock::clone() {
if (getHeaderMask())
NewRegion->createHeaderMask();
+ // The cloned canonical IV increment instruction already carries over its
+ // original NUW flag via the deep clone above, but the new region's
+ // CanIVInfo defaults to NUW set. Keep the two in sync.
+ if (CanIV && !hasCanonicalIVNUW())
+ NewRegion->CanIVInfo->clearNUW();
+
for (VPBlockBase *Block : vp_depth_first_shallow(NewEntry))
Block->setParent(NewRegion);
return NewRegion;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 9a2785a10daf0..891c1fa95750f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug -mtriple=aarch64-linux-gnu -mcpu=neoverse-v1 -S %s | FileCheck %s
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: define void @test_epilogue_tf(
-; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
@@ -32,18 +32,19 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 7
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N_RND_UP]], 8
-; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP3]], align 1
-; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX2]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> splat (i8 1), ptr align 1 [[TMP3]], <8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
More information about the llvm-commits
mailing list