[llvm] [LV][EpilogueTailFolding] hack patch to start by codegen (PR #208764)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 10 09:16:16 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/208764
>From c40ae658f9e4c88b65ef5f5f235f9e5f87ba2598 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 10 Jul 2026 16:06:34 +0100
Subject: [PATCH] [LV][EpilogueTailFolding] hack patch to start by codegen
---
.../Vectorize/LoopVectorizationPlanner.h | 24 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 194 +++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 25 ++-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +
.../Vectorize/VPlanConstruction.cpp | 18 +-
.../AArch64/fold-epilogue-tail.ll | 63 ++++++
.../LoopVectorize/fold-epilogue-tail.ll | 11 +-
7 files changed, 274 insertions(+), 70 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cb861c90beed3..af4359f7e8c3c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -883,16 +883,17 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
- void plan(ElementCount UserVF, unsigned UserIC);
+ void plan(ElementCount UserVF, unsigned UserIC, bool IsEpilogueTFEnabled);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF) const;
+ VPlan &getPlanFor(ElementCount VF, bool TF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
- std::pair<VectorizationFactor, VPlan *> computeBestVF();
+ std::pair<VectorizationFactor, VPlan *>
+ computeBestVF(bool IsEpilogueTFEnabled);
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -918,6 +919,7 @@ class LoopVectorizationPlanner {
DenseMap<const SCEV *, Value *>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
+ bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind =
EpilogueVectorizationKind::None);
@@ -927,10 +929,7 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF) const {
- return any_of(VPlans,
- [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
- }
+ bool hasPlanWithVF(ElementCount VF, bool TF) const;
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
@@ -943,8 +942,10 @@ class LoopVectorizationPlanner {
/// VF narrowed to the chosen factor. The returned plan is a duplicate.
/// Returns nullptr if epilogue vectorization is not supported or not
/// profitable for the loop.
- std::unique_ptr<VPlan>
- selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC);
+ std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
+ ElementCount MainLoopVF,
+ unsigned IC,
+ bool IsEpilogueTFEnabled);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
@@ -981,7 +982,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1();
+ VPlanPtr tryToBuildVPlan1(bool IsEpilogueTFEnabled);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -996,7 +997,8 @@ class LoopVectorizationPlanner {
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ bool IsEpilogueTFEnabled);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 19b77e40a4103..17f6aa6d81599 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3152,6 +3152,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
+ if (!Plan->isCompatibleWithTF(CM.foldTailByMasking()))
+ continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3454,7 +3456,8 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
}
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
- VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
+ VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
+ bool IsEpilogueTFEnabled) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
@@ -3494,9 +3497,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
- if (hasPlanWithVF(EpilogueVectorizationForceVF)) {
+ if (hasPlanWithVF(EpilogueVectorizationForceVF, CM.foldTailByMasking())) {
std::unique_ptr<VPlan> Clone(
- getPlanFor(EpilogueVectorizationForceVF).duplicate());
+ getPlanFor(EpilogueVectorizationForceVF, CM.foldTailByMasking())
+ .duplicate());
Clone->setVF(EpilogueVectorizationForceVF);
return Clone;
}
@@ -3587,10 +3591,12 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width))
+ if (!hasPlanWithVF(NextVF.Width,
+ CM.foldTailByMasking() || IsEpilogueTFEnabled))
continue;
- VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ VPlan &CurrentPlan =
+ getPlanFor(NextVF.Width, CM.foldTailByMasking() || IsEpilogueTFEnabled);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
// vectors) or > the VF of the main loop (fixed vectors).
@@ -3620,7 +3626,8 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
if (Result.Width.isScalar() ||
- isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
+ isMoreProfitable(NextVF, Result, MaxTripCount,
+ !(CM.foldTailByMasking() || IsEpilogueTFEnabled),
/*IsEpilogue*/ true)) {
Result = NextVF;
BestPlan = &CurrentPlan;
@@ -5490,7 +5497,8 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
}
-void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
+void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
+ bool IsEpilogueTFEnabled) {
CM.collectValuesToIgnore();
Config.collectElementTypesForWidening(&CM.ValuesToIgnore);
@@ -5505,7 +5513,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ false);
if (!VPlan1)
return;
@@ -5514,7 +5522,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF);
+ buildVPlans(*VPlan1, VF, VF, /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5557,9 +5565,10 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ false);
}
- buildVPlans(*VPlan1, UserVF, UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
if (!VPlans.empty() && VPlans.back()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
@@ -5590,10 +5599,27 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM.collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
-
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
+
+ // Build tail-folded vplans when IsEpilogueTFEnabled is enabled:
+ if (IsEpilogueTFEnabled) {
+ auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
+ if (!TFVPlan1)
+ return;
+ buildVPlans(*TFVPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ true);
+ buildVPlans(*TFVPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ true);
+ LLVM_DEBUG(dbgs() << "LV: Tail-folded vplans:\n");
+ for (auto &vplan : VPlans) {
+ if (vplan->isCompatibleWithTF(true))
+ LLVM_DEBUG(vplan->dump());
+ }
+ }
}
InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
@@ -5790,7 +5816,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
}
std::pair<VectorizationFactor, VPlan *>
-LoopVectorizationPlanner::computeBestVF() {
+LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
@@ -5800,13 +5826,14 @@ LoopVectorizationPlanner::computeBestVF() {
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
+ assert((FirstPlan.hasScalarVFOnly() ||
+ hasPlanWithVF(UserVF, CM.foldTailByMasking()) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF()) {
+ if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) && hasForcedEpilogueVF()) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
assert(VPlans[0]->getSingleVF() == EpilogueVectorizationForceVF &&
"expected first plan to be for the forced epilogue VF");
@@ -5880,9 +5907,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
- BestFactor = CurrentFactor;
- PlanForBestVF = P.get();
+ if (P->isCompatibleWithTF(CM.foldTailByMasking())) {
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
+ BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
+ }
}
// If profitable add it to ProfitableVF list.
@@ -5902,7 +5931,7 @@ LoopVectorizationPlanner::computeBestVF() {
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
- InnerLoopVectorizer &ILV, DominatorTree *DT,
+ InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind) {
assert(BestVPlan.hasVF(BestVF) &&
"Trying to execute plan with unsupported VF");
@@ -5986,8 +6015,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
assert((OrigLoop->getUniqueLatchExitBlock() || RequiresScalarEpilogue) &&
"loops not exiting via the latch without required epilogue?");
VPlanTransforms::materializeVectorTripCount(
- BestVPlan, VectorPH, CM.foldTailByMasking(), RequiresScalarEpilogue,
- &BestVPlan.getVFxUF(), MaxRuntimeStep);
+ BestVPlan, VectorPH, (CM.foldTailByMasking() || IsEpilogueTFEnabled),
+ RequiresScalarEpilogue, &BestVPlan.getVFxUF(), MaxRuntimeStep);
VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
// Limit expansions to VPInstruction to when not vectorizing the epilogue.
// Currently this code path still relies on code re-using SCEVs expanded
@@ -6497,7 +6526,7 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6576,7 +6605,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM.foldTailByMasking())
+ if (CM.foldTailByMasking() || IsEpilogueTFEnabled)
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6584,7 +6613,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
}
void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF) {
+ ElementCount MaxVF,
+ bool IsEpilogueTFEnabled) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
@@ -6616,6 +6646,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
VPlans.push_back(std::move(P));
TailFoldingStyle Style = CM.getTailFoldingStyle();
+ if (IsEpilogueTFEnabled)
+ Style = TailFoldingStyle::Data;
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -7192,7 +7224,8 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
/// otherwise CM_EpilogueAllowed.
static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
- OptimizationRemarkEmitter *ORE) {
+ OptimizationRemarkEmitter *ORE,
+ LoopVectorizationLegality &LVL) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -7214,6 +7247,13 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (LVL.hasUncountableEarlyExit()) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
+ " of loop has early exit\n"
+ "LV: Fall back to a normal epilogue\n");
+ return CM_EpilogueAllowed;
+ }
+
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
@@ -7754,6 +7794,8 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
for (auto [ResumeV, HeaderPhi] :
zip(ResumeValues, BestEpiPlan.getScalarHeader()->phis())) {
auto *HeaderPhiR = cast<VPIRPhi>(&HeaderPhi);
+ if (!isa<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH)))
+ continue;
auto *EpiResumePhi =
cast<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH));
if (EpiResumePhi->getBasicBlockIndex(BypassBlock) == -1)
@@ -7770,15 +7812,15 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
/// and runtime checks of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
+static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
- DominatorTree *DT,
+ DominatorTree *DT, LoopInfo *LI,
GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove,
- ArrayRef<VPInstruction *> ResumeValues) {
+ ArrayRef<VPInstruction *> ResumeValues,
+ bool IsEpilogueTFEnabled) {
BasicBlock *VecEpilogueIterationCountCheck =
cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
-
BasicBlock *VecEpiloguePreHeader =
cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator())
->getSuccessor(1);
@@ -7802,7 +7844,11 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
BasicBlock *ScalarPH =
cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
- RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
+ // With a tail-folded epilogue there is no scalar remainder to bail
+ // to, even a trip count too small for the epilogue VF is handled safely by
+ // the masked epilogue vector loop, so skip straight to its preheader.
+ RedirectEdge(EPI.EpilogueIterationCountCheck,
+ IsEpilogueTFEnabled ? VecEpiloguePreHeader : ScalarPH);
// Adjust the terminators of runtime check blocks and phis using them.
BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
@@ -7833,11 +7879,20 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
return EPI.EpilogueIterationCountCheck == IncB;
}))
continue;
- for (BasicBlock *BB :
- {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
+ for (BasicBlock *BB : {SCEVCheckBlock, MemCheckBlock}) {
if (BB)
Phi->removeIncomingValue(BB);
}
+ // When the epilogue is tail-folded, EpilogueIterationCountCheck
+ // (iter.check) is redirected to branch straight into the vector epilogue
+ // preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
+ // genuine predecessor and its incoming value must be kept rather than
+ // stripped.
+ // TODO: revisit for reduction phis, whose resume value on this bypass
+ // edge may need dedicated handling rather than reusing the value already
+ // present here.
+ if (!IsEpilogueTFEnabled)
+ Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
}
auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
@@ -7855,6 +7910,47 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
for (PHINode &Phi : make_early_inc_range(VecEpiloguePreHeader->phis()))
if (Phi.use_empty())
Phi.eraseFromParent();
+
+ if (IsEpilogueTFEnabled) {
+ // The epilogue vector loop is tail-folded, so it can safely handle
+ // any remaining trip count, including zero, via masking.
+ // vec.epilog.iter.check's own min-iters check was therefore built with a
+ // compile-time-known-false condition (see
+ // addMinimumVectorEpilogueIterationCheck) that never needs to bail out to
+ // a scalar remainder. Fold it into an unconditional branch into the
+ // vector epilogue preheader.
+ auto *Br =
+ cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator());
+ [[maybe_unused]] auto *CondC = dyn_cast<ConstantInt>(Br->getCondition());
+ assert(CondC && CondC->isZero() &&
+ "expected vec.epilog.iter.check's branch condition to be a "
+ "compile-time false constant when the epilogue is tail-folded");
+ BasicBlock *DeadSucc = Br->getSuccessor(0);
+ UncondBrInst::Create(VecEpiloguePreHeader, Br->getIterator());
+ Br->eraseFromParent();
+ DTU.applyUpdates(
+ {{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
+ }
+
+ if (IsEpilogueTFEnabled && !SCEVCheckBlock && !MemCheckBlock) {
+ // No runtime check still needs a scalar fallback, and every trip-count
+ // bypass edge above has been redirected away from the scalar preheader:
+ // the scalar loop is now entirely unreachable. Delete it outright, along
+ // with its LoopInfo entry, instead of leaving it behind as dead code.
+ // This must run last, since fixScalarResumeValuesFromBypass (above) and
+ // the DT/function verification in processLoop (below) still need `L` and
+ // ScalarPH to be valid up to this point.
+ assert(pred_empty(ScalarPH) &&
+ "scalar preheader should have no predecessors left");
+ auto Blocks = L->getBlocksVector();
+ Blocks.push_back(ScalarPH);
+ while (!L->isInnermost())
+ LI->erase(*L->begin());
+ LI->erase(L);
+ for (auto *BB : Blocks)
+ LI->removeBlock(BB);
+ DeleteDeadBlocks(Blocks, &DTU);
+ }
}
bool LoopVectorizePass::processLoop(Loop *L) {
@@ -8045,15 +8141,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Hints, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE);
+ getEpilogueTailLowering(CM, L, ORE, LVL);
+ bool IsEpilogueTFEnabled = false;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
- reportVectorizationInfo(
- "The epilogue-tail-folding policy prefer-fold-tail is not supported "
- "yet, fall back to a normal epilogue",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
+ IsEpilogueTFEnabled = true;
}
// Get user vectorization factor and interleave count.
@@ -8067,8 +8161,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
UserIC = 1;
// Plan how to best vectorize.
- LVP.plan(UserVF, UserIC);
- auto [VF, BestPlanPtr] = LVP.computeBestVF();
+ LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
+ auto [VF, BestPlanPtr] = LVP.computeBestVF(IsEpilogueTFEnabled);
unsigned IC = 1;
// For VPlan build stress testing of outer loops, bail after plan
@@ -8084,7 +8178,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
CM.maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8146,7 +8240,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) &&
+ UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
@@ -8272,7 +8367,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
std::unique_ptr<VPlan> EpiPlan =
- LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC);
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, IsEpilogueTFEnabled);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
@@ -8305,6 +8400,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Checks, BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
+ /*IsEpilogueTFEnabled*/ false,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
@@ -8336,9 +8432,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LVP.attachRuntimeChecks(BestEpiPlan, Checks, HasBranchWeights);
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
+ IsEpilogueTFEnabled,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
- ResumeValues);
+ connectEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L, EPI, DT, LI, Checks,
+ InstsToMove, ResumeValues, IsEpilogueTFEnabled);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, Checks,
@@ -8350,7 +8447,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (!IsInnerLoop)
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
<< "\"\n");
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT);
+ LVP.executePlan(VF.Width, IC, BestPlan, LB, DT,
+ /*IsEpilogueTFEnabled*/ false);
++LoopsVectorized;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2088f91657289..2727626c71650 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1317,6 +1317,13 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
+bool VPlan::isCompatibleWithTF(bool TF) {
+ auto *VLR = getVectorLoopRegion();
+ assert(VLR && "Vector loop region got eliminated\n");
+ bool HasHeaderMask = (VLR->getHeaderMask() != nullptr);
+ return HasHeaderMask == TF;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1686,19 +1693,29 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
assert(count_if(VPlans,
- [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
- 1 &&
+ [VF, TF](const VPlanPtr &Plan) {
+ LLVM_DEBUG(dbgs() << "LV: given VF: " << VF << " and TF: "
+ << TF << " equivalent vplan: ";
+ Plan->dump());
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ }) == 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF))
+ if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
+bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
+ return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ });
+}
+
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5e722efcf17fa..79413b80d8e20 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5191,6 +5191,15 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
+ /// Returns true if this VPlan's tail-folding matches \p TF.
+ /// A plan is tail-folded if it has a header mask, or if it has no scalar
+ /// tail while the TC is not evenly divisible by VF (in which case the tail
+ /// must be folded by the vector loop itself).
+ ///
+ /// This function is used to disambiguate VPlan lookup by VF when multiple
+ /// plans share the same VF but differ in tail-folding status.
+ bool isCompatibleWithTF(bool TF);
+
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1a135b8549514..6b88b9048cdfa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1582,9 +1582,25 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep,
unsigned EpilogueLoopStep, ScalarEvolution &SE) {
// Add the minimum iteration check for the epilogue vector loop.
+ VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
+
+ if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
+ return;
+ }
+ // if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ // VPBasicBlock *EntryVPBB = cast<VPBasicBlock>(Plan.getEntry());
+ // // Successor 0 is the "taken" (scalar) edge, successor 1 is the vector
+ // // path — see BranchOnCond's execute() and removeBranchOnConst's
+ // // RemovedIdx convention. Drop the scalar edge directly, leaving Entry
+ // // with a single successor (which needs no BranchOnCond terminator at
+ // // all — VPlan codegen emits a plain unconditional `br` for that case).
+ // VPBlockUtils::disconnectBlocks(EntryVPBB, EntryVPBB->getSuccessors()[0]);
+ // return;
+ // }
+
VPValue *TC = Plan.getTripCount();
Value *TripCount = TC->getLiveInIRValue();
- VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
VPValue *VFxUF = Builder.createExpandSCEV(SE.getElementCount(
TripCount->getType(), (EpilogueVF * EpilogueUF), SCEV::FlagNUW));
VPValue *Count = Builder.createSub(TC, Plan.getOrAddLiveIn(VectorTripCount),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
new file mode 100644
index 0000000000000..1cbd4b225efd9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+
+
+define void @test_epilogue_tf(ptr %A, i64 %n) {
+; CHECK-LABEL: define void @test_epilogue_tf(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP0]], align 1
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 15
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 590907a5aff79..6bb1f7349413a 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,15 +1,14 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail\
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 \
+; RUN: opt -S -p loop-vectorize -debug -enable-epilogue-vectorization=false \
+; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize < %s 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: epilogue tail-folding is not supported yet
-; CHECK: remark: <unknown>:0:0: The epilogue-tail-folding policy prefer-fold-tail is not supported yet, fall back to a normal epilogue
+; CHECK: LV: epilogue tail-folding is enabled
;
entry:
br label %for.body
More information about the llvm-commits
mailing list