[llvm] [LV] Support tail-folded epilogue loops (PR #208764)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 07:47:58 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/208764
>From ab7c430bbade7c985a7275bef19e7e7d50463547 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 10 Jul 2026 16:06:34 +0100
Subject: [PATCH 01/16] [LV][EpilogueTailFolding] hack patch to start by
codegen
---
.../Vectorize/LoopVectorizationPlanner.h | 23 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 180 +++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 25 ++-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +
.../Vectorize/VPlanConstruction.cpp | 18 +-
.../AArch64/fold-epilogue-tail.ll | 63 ++++++
.../LoopVectorize/fold-epilogue-tail.ll | 3 +-
7 files changed, 260 insertions(+), 61 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 2e5b0f64bfed2..fee4c5029b50b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -931,16 +931,17 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
- void plan(ElementCount UserVF, unsigned UserIC);
+ void plan(ElementCount UserVF, unsigned UserIC, bool IsEpilogueTFEnabled);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF) const;
+ VPlan &getPlanFor(ElementCount VF, bool TF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
- std::pair<VectorizationFactor, VPlan *> computeBestVF();
+ std::pair<VectorizationFactor, VPlan *>
+ computeBestVF(bool IsEpilogueTFEnabled);
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -966,6 +967,7 @@ class LoopVectorizationPlanner {
DenseMap<const SCEV *, Value *>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
+ bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind =
EpilogueVectorizationKind::None);
@@ -975,10 +977,7 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF) const {
- return any_of(VPlans,
- [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
- }
+ bool hasPlanWithVF(ElementCount VF, bool TF) const;
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
@@ -992,10 +991,13 @@ class LoopVectorizationPlanner {
/// Returns nullptr if epilogue vectorization is not supported or not
/// profitable for the loop. \p ScalarEpilogueAllowed indicates whether the
/// epilogue lowering policy permits creating a scalar epilogue at all.
+ /// \p IsEpilogueTFEnabled indicates whether the epilogue loop should be
+ /// tail-folded rather than left with a scalar epilogue of its own.
std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
ElementCount MainLoopVF,
unsigned IC,
- bool ScalarEpilogueAllowed);
+ bool ScalarEpilogueAllowed,
+ bool IsEpilogueTFEnabled);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
@@ -1029,7 +1031,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1();
+ VPlanPtr tryToBuildVPlan1(bool IsEpilogueTFEnabled);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1044,7 +1046,8 @@ class LoopVectorizationPlanner {
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ bool IsEpilogueTFEnabled);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ba0421869216f..2d17b89db6e90 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3128,6 +3128,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
+ if (!Plan->isCompatibleWithTF(CM->foldTailByMasking()))
+ continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3371,7 +3373,7 @@ bool VFSelectionContext::isEpilogueVectorizationProfitable(
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
- bool ScalarEpilogueAllowed) {
+ bool ScalarEpilogueAllowed, bool IsEpilogueTFEnabled) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
@@ -3411,9 +3413,12 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
- if (hasPlanWithVF(EpilogueVectorizationForceVF)) {
+ if (hasPlanWithVF(EpilogueVectorizationForceVF,
+ MainPlan.hasTailFolded() || IsEpilogueTFEnabled)) {
std::unique_ptr<VPlan> Clone(
- getPlanFor(EpilogueVectorizationForceVF).duplicate());
+ getPlanFor(EpilogueVectorizationForceVF,
+ MainPlan.hasTailFolded() || IsEpilogueTFEnabled)
+ .duplicate());
Clone->setVF(EpilogueVectorizationForceVF);
return Clone;
}
@@ -3504,10 +3509,12 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width))
+ if (!hasPlanWithVF(NextVF.Width,
+ MainPlan.hasTailFolded() || IsEpilogueTFEnabled))
continue;
- VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ VPlan &CurrentPlan = getPlanFor(
+ NextVF.Width, MainPlan.hasTailFolded() || IsEpilogueTFEnabled);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
// the VF of the main loop.
@@ -5402,7 +5409,8 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
}
-void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
+void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
+ bool IsEpilogueTFEnabled) {
CM->collectValuesToIgnore();
Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
@@ -5417,7 +5425,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ false);
if (!VPlan1)
return;
@@ -5426,7 +5434,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF);
+ buildVPlans(*VPlan1, VF, VF, /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5465,12 +5473,13 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
- buildVPlans(*VPlan1, UserVF, UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ false);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
@@ -5502,10 +5511,27 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM->collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
-
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
+
+ // Build tail-folded vplans when IsEpilogueTFEnabled is enabled:
+ if (IsEpilogueTFEnabled) {
+ auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
+ if (!TFVPlan1)
+ return;
+ buildVPlans(*TFVPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
+ /*IsEpilogueTFEnabled*/ true);
+ buildVPlans(*TFVPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ /*IsEpilogueTFEnabled*/ true);
+ LLVM_DEBUG(dbgs() << "LV: Tail-folded vplans:\n");
+ for (auto &vplan : VPlans) {
+ if (vplan->isCompatibleWithTF(true))
+ LLVM_DEBUG(vplan->dump());
+ }
+ }
}
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
@@ -5738,7 +5764,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
}
std::pair<VectorizationFactor, VPlan *>
-LoopVectorizationPlanner::computeBestVF() {
+LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
@@ -5748,13 +5774,15 @@ LoopVectorizationPlanner::computeBestVF() {
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
+ assert((FirstPlan.hasScalarVFOnly() ||
+ hasPlanWithVF(UserVF, CM->foldTailByMasking()) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
+ if (hasPlanWithVF(UserVF, CM->foldTailByMasking()) && hasForcedEpilogueVF() &&
+ VPlans.size() == 2) {
assert(VPlans[0]->getSingleVF() == UserVF &&
"expected second plan to be for the forced UserVF");
assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
@@ -5828,9 +5856,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
- BestFactor = CurrentFactor;
- PlanForBestVF = P.get();
+ if (P->isCompatibleWithTF(CM->foldTailByMasking())) {
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
+ BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
+ }
}
// If profitable add it to ProfitableVF list.
@@ -5863,7 +5893,7 @@ void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
- InnerLoopVectorizer &ILV, DominatorTree *DT,
+ InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind) {
assert(BestVPlan.hasVF(BestVF) &&
"Trying to execute plan with unsupported VF");
@@ -6476,7 +6506,7 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6559,7 +6589,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM->foldTailByMasking())
+ if (CM->foldTailByMasking() || IsEpilogueTFEnabled)
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
@@ -6567,7 +6597,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
}
void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF) {
+ ElementCount MaxVF,
+ bool IsEpilogueTFEnabled) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
@@ -6599,6 +6630,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
VPlans.push_back(std::move(P));
TailFoldingStyle Style = CM->getTailFoldingStyle();
+ if (IsEpilogueTFEnabled)
+ Style = TailFoldingStyle::Data;
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -7756,6 +7789,8 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
for (auto [ResumeV, HeaderPhi] :
zip(ResumeValues, BestEpiPlan.getScalarHeader()->phis())) {
auto *HeaderPhiR = cast<VPIRPhi>(&HeaderPhi);
+ if (!isa<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH)))
+ continue;
auto *EpiResumePhi =
cast<PHINode>(HeaderPhiR->getIRPhi().getIncomingValueForBlock(PH));
if (EpiResumePhi->getBasicBlockIndex(BypassBlock) == -1)
@@ -7772,15 +7807,15 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
/// and runtime checks of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
+static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
- DominatorTree *DT,
+ DominatorTree *DT, LoopInfo *LI,
GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove,
- ArrayRef<VPInstruction *> ResumeValues) {
+ ArrayRef<VPInstruction *> ResumeValues,
+ bool IsEpilogueTFEnabled) {
BasicBlock *VecEpilogueIterationCountCheck =
cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
-
BasicBlock *VecEpiloguePreHeader =
cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator())
->getSuccessor(1);
@@ -7804,7 +7839,11 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
BasicBlock *ScalarPH =
cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
- RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
+ // With a tail-folded epilogue there is no scalar remainder to bail
+ // to, even a trip count too small for the epilogue VF is handled safely by
+ // the masked epilogue vector loop, so skip straight to its preheader.
+ RedirectEdge(EPI.EpilogueIterationCountCheck,
+ IsEpilogueTFEnabled ? VecEpiloguePreHeader : ScalarPH);
// Adjust the terminators of runtime check blocks and phis using them.
BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
@@ -7835,11 +7874,20 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
return EPI.EpilogueIterationCountCheck == IncB;
}))
continue;
- for (BasicBlock *BB :
- {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
+ for (BasicBlock *BB : {SCEVCheckBlock, MemCheckBlock}) {
if (BB)
Phi->removeIncomingValue(BB);
}
+ // When the epilogue is tail-folded, EpilogueIterationCountCheck
+ // (iter.check) is redirected to branch straight into the vector epilogue
+ // preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
+ // genuine predecessor and its incoming value must be kept rather than
+ // stripped.
+ // TODO: revisit for reduction phis, whose resume value on this bypass
+ // edge may need dedicated handling rather than reusing the value already
+ // present here.
+ if (!IsEpilogueTFEnabled)
+ Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
}
auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
@@ -7857,6 +7905,46 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
for (PHINode &Phi : make_early_inc_range(VecEpiloguePreHeader->phis()))
if (Phi.use_empty())
Phi.eraseFromParent();
+
+ if (IsEpilogueTFEnabled) {
+ // The epilogue vector loop is tail-folded, so it can safely handle
+ // any remaining trip count, including zero, via masking.
+ // vec.epilog.iter.check's own min-iters check was therefore built with a
+ // compile-time-known-false condition (see
+ // addMinimumVectorEpilogueIterationCheck) that never needs to bail out to
+ // a scalar remainder. Fold it into an unconditional branch into the
+ // vector epilogue preheader.
+ auto *Br =
+ cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator());
+ [[maybe_unused]] auto *CondC = dyn_cast<ConstantInt>(Br->getCondition());
+ assert(CondC && CondC->isZero() &&
+ "expected vec.epilog.iter.check's branch condition to be a "
+ "compile-time false constant when the epilogue is tail-folded");
+ BasicBlock *DeadSucc = Br->getSuccessor(0);
+ UncondBrInst::Create(VecEpiloguePreHeader, Br->getIterator());
+ Br->eraseFromParent();
+ DTU.applyUpdates(
+ {{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
+ }
+
+ if (IsEpilogueTFEnabled && !SCEVCheckBlock && !MemCheckBlock) {
+ // No runtime check still needs a scalar fallback, and every trip-count
+ // bypass edge above has been redirected away from the scalar preheader:
+ // the scalar loop is now entirely unreachable. Delete it outright, along
+ // with its LoopInfo entry, instead of leaving it behind as dead code.
+ // This must run last, since fixScalarResumeValuesFromBypass (above) and
+ // the DT/function verification in processLoop (below) still need `L` and
+ // ScalarPH to be valid up to this point.
+ assert(pred_empty(ScalarPH) &&
+ "scalar preheader should have no predecessors left");
+ SmallVector<BasicBlock *> Blocks(L->block_begin(),
+ L->block_end());
+ Blocks.push_back(ScalarPH);
+ LI->erase(L);
+ for (auto *BB : Blocks)
+ LI->removeBlock(BB);
+ DeleteDeadBlocks(Blocks, &DTU);
+ }
}
bool LoopVectorizePass::processLoop(Loop *L) {
@@ -8049,14 +8137,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
+ bool IsEpilogueTFEnabled = false;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
- reportVectorizationInfo(
- "The epilogue-tail-folding policy prefer-fold-tail is not supported "
- "yet, fall back to a normal epilogue",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
+ IsEpilogueTFEnabled = true;
}
// Get user vectorization factor and interleave count.
@@ -8070,8 +8156,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
UserIC = 1;
// Plan how to best vectorize.
- LVP.plan(UserVF, UserIC);
- auto [VF, BestPlanPtr] = LVP.computeBestVF();
+ LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
+ auto [VF, BestPlanPtr] = LVP.computeBestVF(IsEpilogueTFEnabled);
unsigned IC = 1;
// For VPlan build stress testing of outer loops, bail after plan
@@ -8087,7 +8173,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
LVP.getCostModel().maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width,
+ LVP.getCostModel().foldTailByMasking())) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8149,7 +8236,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width,
+ LVP.getCostModel().foldTailByMasking()) &&
+ UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
@@ -8283,8 +8372,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
- std::unique_ptr<VPlan> EpiPlan =
- LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
+ std::unique_ptr<VPlan> EpiPlan = LVP.selectBestEpiloguePlan(
+ BestPlan, VF.Width, IC, ScalarEpilogueAllowed, IsEpilogueTFEnabled);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
@@ -8316,6 +8405,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
+ /*IsEpilogueTFEnabled*/ false,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
@@ -8347,9 +8437,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
+ IsEpilogueTFEnabled,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
- ResumeValues);
+ connectEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L, EPI, DT, LI, Checks,
+ InstsToMove, ResumeValues, IsEpilogueTFEnabled);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
@@ -8361,7 +8452,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (!IsInnerLoop)
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
<< "\"\n");
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT);
+ LVP.executePlan(VF.Width, IC, BestPlan, LB, DT,
+ /*IsEpilogueTFEnabled*/ false);
++LoopsVectorized;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 87d24ed9d8d4b..b0ffebcae1a20 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1344,6 +1344,13 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
+bool VPlan::isCompatibleWithTF(bool TF) {
+ auto *VLR = getVectorLoopRegion();
+ assert(VLR && "Vector loop region got eliminated\n");
+ bool HasHeaderMask = (VLR->getHeaderMask() != nullptr);
+ return HasHeaderMask == TF;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1713,19 +1720,29 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
assert(count_if(VPlans,
- [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
- 1 &&
+ [VF, TF](const VPlanPtr &Plan) {
+ LLVM_DEBUG(dbgs() << "LV: given VF: " << VF << " and TF: "
+ << TF << " equivalent vplan: ";
+ Plan->dump());
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ }) == 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF))
+ if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
+bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
+ return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ });
+}
+
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 630f6036ea535..d7783f2fce7da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5250,6 +5250,15 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
+ /// Returns true if this VPlan's tail-folding matches \p TF.
+ /// A plan is tail-folded if it has a header mask, or if it has no scalar
+ /// tail while the TC is not evenly divisible by VF (in which case the tail
+ /// must be folded by the vector loop itself).
+ ///
+ /// This function is used to disambiguate VPlan lookup by VF when multiple
+ /// plans share the same VF but differ in tail-folding status.
+ bool isCompatibleWithTF(bool TF);
+
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a2006ebf4bb4f..476f1898a61ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1540,9 +1540,25 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep,
unsigned EpilogueLoopStep, ScalarEvolution &SE) {
// Add the minimum iteration check for the epilogue vector loop.
+ VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
+
+ if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
+ return;
+ }
+ // if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ // VPBasicBlock *EntryVPBB = cast<VPBasicBlock>(Plan.getEntry());
+ // // Successor 0 is the "taken" (scalar) edge, successor 1 is the vector
+ // // path — see BranchOnCond's execute() and removeBranchOnConst's
+ // // RemovedIdx convention. Drop the scalar edge directly, leaving Entry
+ // // with a single successor (which needs no BranchOnCond terminator at
+ // // all — VPlan codegen emits a plain unconditional `br` for that case).
+ // VPBlockUtils::disconnectBlocks(EntryVPBB, EntryVPBB->getSuccessors()[0]);
+ // return;
+ // }
+
VPValue *TC = Plan.getTripCount();
Value *TripCount = TC->getLiveInIRValue();
- VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
VPValue *VFxUF = Builder.createExpandSCEV(SE.getElementCount(
TripCount->getType(), (EpilogueVF * EpilogueUF), SCEV::FlagNUW));
VPValue *Count = Builder.createSub(TC, Plan.getOrAddLiveIn(VectorTripCount),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
new file mode 100644
index 0000000000000..495b280990aec
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+
+
+define void @test_epilogue_tf(ptr %A, i64 %n) {
+; CHECK-LABEL: define void @test_epilogue_tf(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP0]], align 1
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 7
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N_RND_UP]], 7
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
+; CHECK-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 78aa3eda9d952..b56b4c622fd5a 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -23,8 +23,7 @@
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: epilogue tail-folding is not supported yet
-; CHECK: remark: <unknown>:0:0: The epilogue-tail-folding policy prefer-fold-tail is not supported yet, fall back to a normal epilogue
+; CHECK: LV: epilogue tail-folding is enabled
;
; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
>From 40e18250da567f16594f39d1ad8ded0702efa1d6 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 21 Jul 2026 15:10:26 +0100
Subject: [PATCH 02/16] rebase and resolve review comments - limit the feature
to be enabled only for forced epilogue VF
---
.../Vectorize/LoopVectorizationPlanner.h | 10 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 65 ++++++-------------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 25 ++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 9 ---
.../Vectorize/VPlanConstruction.cpp | 12 +---
.../AArch64/fold-epilogue-tail.ll | 3 +-
6 files changed, 33 insertions(+), 91 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fee4c5029b50b..ef4b84e1d82c0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -935,13 +935,12 @@ class LoopVectorizationPlanner {
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF, bool TF) const;
+ VPlan &getPlanFor(ElementCount VF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
- std::pair<VectorizationFactor, VPlan *>
- computeBestVF(bool IsEpilogueTFEnabled);
+ std::pair<VectorizationFactor, VPlan *> computeBestVF();
/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
@@ -977,7 +976,10 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF, bool TF) const;
+ bool hasPlanWithVF(ElementCount VF) const {
+ return any_of(VPlans,
+ [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
+ }
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2d17b89db6e90..2bf199714613e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3128,8 +3128,6 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
- if (!Plan->isCompatibleWithTF(CM->foldTailByMasking()))
- continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3413,12 +3411,9 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
- if (hasPlanWithVF(EpilogueVectorizationForceVF,
- MainPlan.hasTailFolded() || IsEpilogueTFEnabled)) {
+ if (hasPlanWithVF(EpilogueVectorizationForceVF)) {
std::unique_ptr<VPlan> Clone(
- getPlanFor(EpilogueVectorizationForceVF,
- MainPlan.hasTailFolded() || IsEpilogueTFEnabled)
- .duplicate());
+ getPlanFor(EpilogueVectorizationForceVF).duplicate());
Clone->setVF(EpilogueVectorizationForceVF);
return Clone;
}
@@ -3509,12 +3504,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width,
- MainPlan.hasTailFolded() || IsEpilogueTFEnabled))
+ if (!hasPlanWithVF(NextVF.Width))
continue;
- VPlan &CurrentPlan = getPlanFor(
- NextVF.Width, MainPlan.hasTailFolded() || IsEpilogueTFEnabled);
+ VPlan &CurrentPlan = getPlanFor(NextVF.Width);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
// the VF of the main loop.
@@ -5478,8 +5471,13 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ false);
+ if (IsEpilogueTFEnabled) {
+ auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
+ buildVPlans(*TFVPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ true);
+ } else
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
+ /*IsEpilogueTFEnabled*/ false);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
@@ -5516,22 +5514,6 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
/*IsEpilogueTFEnabled*/ false);
LLVM_DEBUG(printPlans(dbgs()));
-
- // Build tail-folded vplans when IsEpilogueTFEnabled is enabled:
- if (IsEpilogueTFEnabled) {
- auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
- if (!TFVPlan1)
- return;
- buildVPlans(*TFVPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
- /*IsEpilogueTFEnabled*/ true);
- buildVPlans(*TFVPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
- /*IsEpilogueTFEnabled*/ true);
- LLVM_DEBUG(dbgs() << "LV: Tail-folded vplans:\n");
- for (auto &vplan : VPlans) {
- if (vplan->isCompatibleWithTF(true))
- LLVM_DEBUG(vplan->dump());
- }
- }
}
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
@@ -5764,7 +5746,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
}
std::pair<VectorizationFactor, VPlan *>
-LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
+LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
@@ -5774,15 +5756,13 @@ LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() ||
- hasPlanWithVF(UserVF, CM->foldTailByMasking()) ||
+ assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF, CM->foldTailByMasking()) && hasForcedEpilogueVF() &&
- VPlans.size() == 2) {
+ if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
assert(VPlans[0]->getSingleVF() == UserVF &&
"expected second plan to be for the forced UserVF");
assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
@@ -5856,11 +5836,9 @@ LoopVectorizationPlanner::computeBestVF(bool IsEpilogueTFEnabled) {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (P->isCompatibleWithTF(CM->foldTailByMasking())) {
- if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
- BestFactor = CurrentFactor;
- PlanForBestVF = P.get();
- }
+ if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
+ BestFactor = CurrentFactor;
+ PlanForBestVF = P.get();
}
// If profitable add it to ProfitableVF list.
@@ -8157,7 +8135,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
- auto [VF, BestPlanPtr] = LVP.computeBestVF(IsEpilogueTFEnabled);
+ auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
// For VPlan build stress testing of outer loops, bail after plan
@@ -8173,8 +8151,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
LVP.getCostModel().maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width,
- LVP.getCostModel().foldTailByMasking())) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8236,9 +8213,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width,
- LVP.getCostModel().foldTailByMasking()) &&
- UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index b0ffebcae1a20..87d24ed9d8d4b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1344,13 +1344,6 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
-bool VPlan::isCompatibleWithTF(bool TF) {
- auto *VLR = getVectorLoopRegion();
- assert(VLR && "Vector loop region got eliminated\n");
- bool HasHeaderMask = (VLR->getHeaderMask() != nullptr);
- return HasHeaderMask == TF;
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1720,29 +1713,19 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
assert(count_if(VPlans,
- [VF, TF](const VPlanPtr &Plan) {
- LLVM_DEBUG(dbgs() << "LV: given VF: " << VF << " and TF: "
- << TF << " equivalent vplan: ";
- Plan->dump());
- return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
- }) == 1 &&
+ [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
+ 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
+ if (Plan->hasVF(VF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
-bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
- return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
- return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
- });
-}
-
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d7783f2fce7da..630f6036ea535 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5250,15 +5250,6 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
- /// Returns true if this VPlan's tail-folding matches \p TF.
- /// A plan is tail-folded if it has a header mask, or if it has no scalar
- /// tail while the TC is not evenly divisible by VF (in which case the tail
- /// must be folded by the vector loop itself).
- ///
- /// This function is used to disambiguate VPlan lookup by VF when multiple
- /// plans share the same VF but differ in tail-folding status.
- bool isCompatibleWithTF(bool TF);
-
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 476f1898a61ce..3e6f8add8e035 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1542,20 +1542,10 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
// Add the minimum iteration check for the epilogue vector loop.
VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
- if (Plan.isCompatibleWithTF(/*TF*/ true)) {
+ if (Plan.hasTailFolded()) {
Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
return;
}
- // if (Plan.isCompatibleWithTF(/*TF*/ true)) {
- // VPBasicBlock *EntryVPBB = cast<VPBasicBlock>(Plan.getEntry());
- // // Successor 0 is the "taken" (scalar) edge, successor 1 is the vector
- // // path — see BranchOnCond's execute() and removeBranchOnConst's
- // // RemovedIdx convention. Drop the scalar edge directly, leaving Entry
- // // with a single successor (which needs no BranchOnCond terminator at
- // // all — VPlan codegen emits a plain unconditional `br` for that case).
- // VPBlockUtils::disconnectBlocks(EntryVPBB, EntryVPBB->getSuccessors()[0]);
- // return;
- // }
VPValue *TC = Plan.getTripCount();
Value *TripCount = TC->getLiveInIRValue();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 495b280990aec..a94cf64668ff6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
define void @test_epilogue_tf(ptr %A, i64 %n) {
>From 202b23d2e4aa777ff8d09ee18ffe5fb08127da8f Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 23 Jul 2026 11:57:23 +0100
Subject: [PATCH 03/16] Enable correct costs for tail-folded epilogue. To
achieve that, I pass the epilogue CM to cost/vplan-related functions.
---
.../Vectorize/LoopVectorizationPlanner.h | 25 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 242 ++++++++++++------
.../AArch64/fold-epilogue-tail.ll | 37 +--
3 files changed, 195 insertions(+), 109 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ef4b84e1d82c0..7e058d46ef41f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -901,7 +901,8 @@ class LoopVectorizationPlanner {
///
/// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
/// been retired.
- InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
+ InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const;
/// Precompute costs for certain instructions using the legacy cost model. The
/// function is used to bring up the VPlan-based cost model to initially avoid
@@ -931,7 +932,14 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
- void plan(ElementCount UserVF, unsigned UserIC, bool IsEpilogueTFEnabled);
+ void plan(ElementCount UserVF, unsigned UserIC);
+
+ /// Build VPlans for the specified \p EpilogueUserVF and \p IC if they are
+ /// non-zero or all applicable candidate VFs otherwise. If vectorization and
+ /// tail-folding should be avoided up-front, no plans are generated.
+ bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC,
+ ElementCount EpilogueUserVF,
+ LoopVectorizationCostModel &EpilogueCM);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
@@ -966,7 +974,6 @@ class LoopVectorizationPlanner {
DenseMap<const SCEV *, Value *>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
- bool IsEpilogueTFEnabled,
EpilogueVectorizationKind EpilogueVecKind =
EpilogueVectorizationKind::None);
@@ -993,13 +1000,10 @@ class LoopVectorizationPlanner {
/// Returns nullptr if epilogue vectorization is not supported or not
/// profitable for the loop. \p ScalarEpilogueAllowed indicates whether the
/// epilogue lowering policy permits creating a scalar epilogue at all.
- /// \p IsEpilogueTFEnabled indicates whether the epilogue loop should be
- /// tail-folded rather than left with a scalar epilogue of its own.
std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
ElementCount MainLoopVF,
unsigned IC,
- bool ScalarEpilogueAllowed,
- bool IsEpilogueTFEnabled);
+ bool ScalarEpilogueAllowed);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
@@ -1033,7 +1037,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1(bool IsEpilogueTFEnabled);
+ VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1043,13 +1047,14 @@ class LoopVectorizationPlanner {
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
/// original scalar loop.
- VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
+ VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
+ LoopVectorizationCostModel &EnabledCM);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
- bool IsEpilogueTFEnabled);
+ LoopVectorizationCostModel &EnabledCM);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2bf199714613e..976cc47c1f2ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3371,7 +3371,7 @@ bool VFSelectionContext::isEpilogueVectorizationProfitable(
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
- bool ScalarEpilogueAllowed, bool IsEpilogueTFEnabled) {
+ bool ScalarEpilogueAllowed) {
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
return nullptr;
@@ -3608,7 +3608,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (VF.isScalar())
LoopCost = CM->expectedCost(VF);
else
- LoopCost = cost(Plan, VF, &R);
+ LoopCost = cost(Plan, VF, &R, *CM);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
// Loop body is free and there is no need for interleaving.
@@ -5402,8 +5402,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
}
-void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
- bool IsEpilogueTFEnabled) {
+void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM->collectValuesToIgnore();
Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
@@ -5418,7 +5417,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ false);
+ auto VPlan1 = tryToBuildVPlan1(*CM);
if (!VPlan1)
return;
@@ -5427,7 +5426,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF, /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, VF, VF, *CM);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5466,24 +5465,19 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
- buildVPlans(*VPlan1, UserVF, UserVF, /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, UserVF, UserVF, *CM);
+
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- if (IsEpilogueTFEnabled) {
- auto TFVPlan1 = tryToBuildVPlan1(/*IsEpilogueTFEnabled*/ true);
- buildVPlans(*TFVPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ true);
- } else
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF,
- /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
if (UserVF.isScalar() ||
- cost(*VPlans.front(), UserVF, /*RU=*/nullptr).isValid()) {
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid()) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
LLVM_DEBUG(printPlans(dbgs()));
return;
@@ -5509,13 +5503,74 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
CM->collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF,
- /*IsEpilogueTFEnabled*/ false);
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, *CM);
buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
- /*IsEpilogueTFEnabled*/ false);
+ *CM);
+
LLVM_DEBUG(printPlans(dbgs()));
}
+bool LoopVectorizationPlanner::planForEpilogueTF(
+ ElementCount UserVF, unsigned UserIC, ElementCount EpilogueUserVF,
+ LoopVectorizationCostModel &EpilogueCM) {
+ if (VPlans.empty())
+ return false;
+ if (!OrigLoop->isInnermost())
+ return false;
+
+ if (!EpilogueUserVF.isVector() ||
+ ElementCount::isKnownGE(EpilogueUserVF, UserVF))
+ return false;
+
+ EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
+ EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
+
+ FixedScalableVFPair MaxFactors =
+ EpilogueCM.computeMaxVF(EpilogueUserVF, UserIC);
+ if (!MaxFactors ||
+ !EpilogueCM.foldTailByMasking()) // Cases that should not to be vectorized
+ // nor tail-folded.
+ return false;
+
+ auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
+ if (!VPlan1)
+ return false;
+
+ // Invalidate interleave groups if all blocks of loop will be predicated.
+ if (EpilogueCM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ !useMaskedInterleavedAccesses(TTI)) {
+ LLVM_DEBUG(
+ dbgs() << "LV: [EpilogueTF] Invalidate all interleaved groups due to "
+ "fold-tail "
+ "by masking which requires masked-interleaved support.\n");
+ if (EpilogueCM.InterleaveInfo.invalidateGroups())
+ // Invalidating interleave groups also requires invalidating all decisions
+ // based on them, which includes widening decisions and uniform and scalar
+ // values.
+ EpilogueCM.invalidateCostModelingDecisions();
+ }
+
+ if (EpilogueCM.foldTailByMasking())
+ Legal->prepareToFoldTailByMasking();
+
+ // Collect the instructions (and their associated costs) that will be more
+ // profitable to scalarize.
+ EpilogueCM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+
+ assert(VPlans.size() == 2 &&
+ "For tail-folded epilogue, VPlans size is expected to be 2");
+ // remove last vplan which should be the epilogue plan to replace it by our
+ // tail-folded vplan:
+ assert(VPlans.back()->getSingleVF() == EpilogueUserVF &&
+ "For tail-folded epilogue, first vplan is expected to have "
+ "EpilogueUserVF");
+ VPlans.pop_back();
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, EpilogueCM);
+
+ cost(*VPlans.back(), EpilogueUserVF, /*RU=*/nullptr, EpilogueCM);
+ return true;
+}
+
VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
LoopVectorizationCostModel &CM,
VFSelectionContext &Config,
@@ -5708,9 +5763,10 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return Cost;
}
-InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
- VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, *CM, Config,
+InstructionCost LoopVectorizationPlanner::cost(
+ VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const {
+ VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -5833,7 +5889,7 @@ LoopVectorizationPlanner::computeBestVF() {
}
InstructionCost Cost =
- cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
+ cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, *CM);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
@@ -5871,7 +5927,7 @@ void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
- InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueTFEnabled,
+ InnerLoopVectorizer &ILV, DominatorTree *DT,
EpilogueVectorizationKind EpilogueVecKind) {
assert(BestVPlan.hasVF(BestVF) &&
"Trying to execute plan with unsupported VF");
@@ -6484,7 +6540,8 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
+ LoopVectorizationCostModel &EnabledCM) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6535,8 +6592,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6567,24 +6624,24 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(bool IsEpilogueTFEnabled) {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (CM->foldTailByMasking() || IsEpilogueTFEnabled)
+ if (EnabledCM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
}
-void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF,
- bool IsEpilogueTFEnabled) {
+void LoopVectorizationPlanner::buildVPlans(
+ VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- auto Plan =
- tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()), SubRange);
+ auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
+ SubRange, EnabledCM);
VF = SubRange.End;
if (!Plan)
@@ -6597,7 +6654,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (CM->foldTailWithEVL()) {
+ if (EnabledCM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6607,9 +6664,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = CM->getTailFoldingStyle();
- if (IsEpilogueTFEnabled)
- Style = TailFoldingStyle::Data;
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6620,8 +6675,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
}
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
- VFRange &Range) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
+ VPlanPtr Plan, VFRange &Range, LoopVectorizationCostModel &EnabledCM) {
// For outer loops, the plan only needs basic recipe conversion and induction
// live-out optimization; the full inner-loop recipe building below does not
@@ -6647,8 +6702,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
- [this](ElementCount VF) {
- return !CM->requiresScalarEpilogue(VF.isVector());
+ [&EnabledCM](ElementCount VF) {
+ return !EnabledCM.requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6666,9 +6721,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(CM.get(), VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&EnabledCM, VF);
- TailFoldingStyle Style = CM->getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6694,10 +6749,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Range, add it to the set of groups to be later applied to the VPlan and add
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
- for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
- auto ApplyIG = [IG, this](ElementCount VF) -> bool {
+ for (InterleaveGroup<Instruction> *IG :
+ EnabledCM.InterleaveInfo.getInterleaveGroups()) {
+ auto ApplyIG = [IG, &EnabledCM](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- CM->getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM.getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6714,7 +6770,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, EnabledCM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6725,7 +6781,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6832,7 +6888,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, CM->isEpilogueAllowed());
+ InterleaveGroups, EnabledCM.isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6849,7 +6905,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (CM->maskPartialAliasing())
+ if (EnabledCM.maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -7567,7 +7623,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
assert(all_of(IV->users(),
[](const VPUser *U) {
- if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
+ if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe,
+ VPWidenCanonicalIVRecipe>(U))
return true;
unsigned Opc = cast<VPInstruction>(U)->getOpcode();
return Instruction::isCast(Opc) || Opc == Instruction::Add;
@@ -7581,7 +7638,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto *Increment = vputils::findCanonicalIVIncrement(Plan);
assert(Increment && "Must have a canonical IV increment at this point");
IV->replaceUsesWithIf(Add, [Add, Increment](VPUser &U, unsigned) {
- return &U != Add && &U != Increment;
+ return &U != Add && &U != Increment && !isa<VPWidenCanonicalIVRecipe>(&U);
});
VPInstruction *OffsetIVInc =
VPBuilder::getToInsertAfter(Increment).createAdd(Increment, VPV);
@@ -7692,6 +7749,23 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
continue;
}
}
+ } else if (isa<VPActiveLaneMaskPHIRecipe>(&R)) {
+ // The active-lane-mask phi's entry value was computed assuming the
+ // epilogue vector loop starts at 0. Rebuild it using the resume value
+ // instead, so the mask reflects how many elements the main vector loop
+ // already processed.
+ VPBuilder EntryBuilder(Plan.getVectorPreheader());
+ Type *CanIVTy = VectorLoop->getCanonicalIVType();
+ VPValue *ALMMultiplier = Plan.getConstantInt(CanIVTy, 1);
+ auto *EntryIncrement = EntryBuilder.createOverflowingOp(
+ VPInstruction::CanonicalIVIncrementForPart, {VPV, &Plan.getVF()}, {},
+ R.getDebugLoc(), "index.part.next");
+ auto *EntryALM = EntryBuilder.createNaryOp(
+ VPInstruction::ActiveLaneMask,
+ {EntryIncrement, Plan.getTripCount(), ALMMultiplier}, R.getDebugLoc(),
+ "active.lane.mask.entry");
+ cast<VPHeaderPHIRecipe>(&R)->setStartValue(EntryALM);
+ continue;
} else {
// Retrieve the induction resume value via ResumeForEpilogue.
PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
@@ -7785,7 +7859,7 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
/// and runtime checks of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
+static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
DominatorTree *DT, LoopInfo *LI,
GeneratedRTChecks &Checks,
@@ -7859,8 +7933,7 @@ static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
// When the epilogue is tail-folded, EpilogueIterationCountCheck
// (iter.check) is redirected to branch straight into the vector epilogue
// preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
- // genuine predecessor and its incoming value must be kept rather than
- // stripped.
+ // predecessor and its incoming value must be kept rather than stripped.
// TODO: revisit for reduction phis, whose resume value on this bypass
// edge may need dedicated handling rather than reusing the value already
// present here.
@@ -7903,25 +7976,18 @@ static void connectEpilogueVectorLoop(VPlan &MainPlan, VPlan &EpiPlan, Loop *L,
Br->eraseFromParent();
DTU.applyUpdates(
{{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
- }
- if (IsEpilogueTFEnabled && !SCEVCheckBlock && !MemCheckBlock) {
- // No runtime check still needs a scalar fallback, and every trip-count
- // bypass edge above has been redirected away from the scalar preheader:
- // the scalar loop is now entirely unreachable. Delete it outright, along
- // with its LoopInfo entry, instead of leaving it behind as dead code.
- // This must run last, since fixScalarResumeValuesFromBypass (above) and
- // the DT/function verification in processLoop (below) still need `L` and
- // ScalarPH to be valid up to this point.
- assert(pred_empty(ScalarPH) &&
- "scalar preheader should have no predecessors left");
- SmallVector<BasicBlock *> Blocks(L->block_begin(),
- L->block_end());
- Blocks.push_back(ScalarPH);
- LI->erase(L);
- for (auto *BB : Blocks)
- LI->removeBlock(BB);
- DeleteDeadBlocks(Blocks, &DTU);
+ if (!SCEVCheckBlock && !MemCheckBlock) {
+ // Delete the scalar loop as it's dead right
+ assert(pred_empty(ScalarPH) &&
+ "scalar preheader should have no predecessors left");
+ SmallVector<BasicBlock *> Blocks(L->block_begin(), L->block_end());
+ Blocks.push_back(ScalarPH);
+ LI->erase(L);
+ for (auto *BB : Blocks)
+ LI->removeBlock(BB);
+ DeleteDeadBlocks(Blocks, &DTU);
+ }
}
}
@@ -8116,11 +8182,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
bool IsEpilogueTFEnabled = false;
+ std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
+ std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
- // TODO: Apply tail-folding on the vectorized epilogue loop.
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
IsEpilogueTFEnabled = true;
+ TailFoldingCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
+ if (UseInterleaved && useMaskedInterleavedAccesses(*TTI))
+ TailFoldingCMIAI->analyzeInterleaving(
+ /*useMaskedInterleavedAccesses*/ true);
+ EpilogueTailFoldingCM.emplace(CM_EpilogueNotNeededFoldTail, L, PSE, LI,
+ &LVL, *TTI, TLI, AC, ORE, GetBFI, F,
+ *TailFoldingCMIAI, Config);
}
// Get user vectorization factor and interleave count.
@@ -8134,7 +8208,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
UserIC = 1;
// Plan how to best vectorize.
- LVP.plan(UserVF, UserIC, IsEpilogueTFEnabled);
+ LVP.plan(UserVF, UserIC);
+ if (IsEpilogueTFEnabled)
+ if (!LVP.planForEpilogueTF(UserVF, /*UserIC*/ 1,
+ EpilogueVectorizationForceVF,
+ *EpilogueTailFoldingCM)) {
+ // we can't apply epilogue TF:
+ EpilogueTailFoldingCM.reset();
+ IsEpilogueTFEnabled = false;
+ }
+
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
@@ -8348,7 +8431,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
std::unique_ptr<VPlan> EpiPlan = LVP.selectBestEpiloguePlan(
- BestPlan, VF.Width, IC, ScalarEpilogueAllowed, IsEpilogueTFEnabled);
+ BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
@@ -8380,7 +8463,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BestMainPlan);
auto ExpandedSCEVs = LVP.executePlan(
EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT,
- /*IsEpilogueTFEnabled*/ false,
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
@@ -8412,10 +8494,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
- IsEpilogueTFEnabled,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L, EPI, DT, LI, Checks,
- InstsToMove, ResumeValues, IsEpilogueTFEnabled);
+ connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
+ ResumeValues, IsEpilogueTFEnabled);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
@@ -8427,8 +8508,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (!IsInnerLoop)
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
<< "\"\n");
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT,
- /*IsEpilogueTFEnabled*/ false);
+ LVP.executePlan(VF.Width, IC, BestPlan, LB, DT);
++LoopsVectorized;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index a94cf64668ff6..746e4495e059e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mtriple=aarch64-linux-gnu -S %s | FileCheck %s
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug -mtriple=aarch64-linux-gnu -mcpu=neoverse-v1 -S %s | FileCheck %s
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: define void @test_epilogue_tf(
-; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
@@ -13,18 +13,18 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
-; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP1]], align 1
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP2]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -32,17 +32,18 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 7
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N_RND_UP]], 7
-; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP3]], align 1
-; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX2]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> splat (i8 1), ptr align 1 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
>From b56ed9c255354ae0821d1573e2f4cffdcec83773 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 30 Jul 2026 17:41:23 +0100
Subject: [PATCH 04/16] resolve review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 65 ++++++++++---------
.../Vectorize/VPlanConstruction.cpp | 2 +
.../AArch64/fold-epilogue-tail.ll | 33 +++++++++-
.../LoopVectorize/fold-epilogue-tail.ll | 55 +++++++++++++++-
4 files changed, 123 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 976cc47c1f2ad..ca4cde7caa08e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5465,6 +5465,9 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
+ // Build the main-loop VPlan firstly because if epilogue tail-folding is
+ // enabled, it will be built later, so we keep the epilogue vplans at the
+ // end.
buildVPlans(*VPlan1, UserVF, UserVF, *CM);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
@@ -5513,56 +5516,61 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
bool LoopVectorizationPlanner::planForEpilogueTF(
ElementCount UserVF, unsigned UserIC, ElementCount EpilogueUserVF,
LoopVectorizationCostModel &EpilogueCM) {
- if (VPlans.empty())
+ if (VPlans.empty()) {
+ LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
+ "out of epilogue tail-folding\n");
return false;
+ }
if (!OrigLoop->isInnermost())
return false;
if (!EpilogueUserVF.isVector() ||
- ElementCount::isKnownGE(EpilogueUserVF, UserVF))
+ (estimateElementCount(EpilogueUserVF, Config.getVScaleForTuning()) >=
+ estimateElementCount(UserVF, Config.getVScaleForTuning())) *
+ UserIC)
return false;
EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EpilogueCM.computeMaxVF(EpilogueUserVF, UserIC);
+ EpilogueCM.computeMaxVF(EpilogueUserVF, /*UserIC*/ 1);
if (!MaxFactors ||
- !EpilogueCM.foldTailByMasking()) // Cases that should not to be vectorized
- // nor tail-folded.
+ !EpilogueCM
+ .foldTailByMasking()) { // Cases that should not to be vectorized
+ // or tail-folded.
+ reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
+ "InvalidTailFoldedEpilogue", ORE, OrigLoop);
return false;
+ }
auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
if (!VPlan1)
return false;
- // Invalidate interleave groups if all blocks of loop will be predicated.
- if (EpilogueCM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
- !useMaskedInterleavedAccesses(TTI)) {
+ if (!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
- dbgs() << "LV: [EpilogueTF] Invalidate all interleaved groups due to "
- "fold-tail "
- "by masking which requires masked-interleaved support.\n");
+ dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by "
+ "masking which requires masked-interleaved support.\n");
if (EpilogueCM.InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
EpilogueCM.invalidateCostModelingDecisions();
}
-
- if (EpilogueCM.foldTailByMasking())
- Legal->prepareToFoldTailByMasking();
+ Legal->prepareToFoldTailByMasking();
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
EpilogueCM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ // Expecting only 2 vplans as epilogue TF is applied only for forced VFs.
assert(VPlans.size() == 2 &&
"For tail-folded epilogue, VPlans size is expected to be 2");
- // remove last vplan which should be the epilogue plan to replace it by our
- // tail-folded vplan:
+ // Remove the last vplan, which should be the epilogue plan to replace it by
+ // the tail-folded vplan:
assert(VPlans.back()->getSingleVF() == EpilogueUserVF &&
- "For tail-folded epilogue, first vplan is expected to have "
+ "For tail-folded epilogue, last vplan is expected to have "
"EpilogueUserVF");
VPlans.pop_back();
buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, EpilogueCM);
@@ -7959,7 +7967,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
if (IsEpilogueTFEnabled) {
// The epilogue vector loop is tail-folded, so it can safely handle
- // any remaining trip count, including zero, via masking.
+ // any remaining iterations, including zero, via masking.
// vec.epilog.iter.check's own min-iters check was therefore built with a
// compile-time-known-false condition (see
// addMinimumVectorEpilogueIterationCheck) that never needs to bail out to
@@ -7978,7 +7986,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
{{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
if (!SCEVCheckBlock && !MemCheckBlock) {
- // Delete the scalar loop as it's dead right
+ // Delete the scalar loop as it's dead right now.
assert(pred_empty(ScalarPH) &&
"scalar preheader should have no predecessors left");
SmallVector<BasicBlock *> Blocks(L->block_begin(), L->block_end());
@@ -8181,17 +8189,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
- bool IsEpilogueTFEnabled = false;
std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
- IsEpilogueTFEnabled = true;
TailFoldingCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
- if (UseInterleaved && useMaskedInterleavedAccesses(*TTI))
- TailFoldingCMIAI->analyzeInterleaving(
- /*useMaskedInterleavedAccesses*/ true);
+ if (UseInterleaved)
+ TailFoldingCMIAI->analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
EpilogueTailFoldingCM.emplace(CM_EpilogueNotNeededFoldTail, L, PSE, LI,
&LVL, *TTI, TLI, AC, ORE, GetBFI, F,
*TailFoldingCMIAI, Config);
@@ -8209,13 +8214,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- if (IsEpilogueTFEnabled)
- if (!LVP.planForEpilogueTF(UserVF, /*UserIC*/ 1,
- EpilogueVectorizationForceVF,
- *EpilogueTailFoldingCM)) {
+ if (EpilogueTailFoldingCM.has_value())
+ if (!LVP.planForEpilogueTF(UserVF, UserIC, EpilogueVectorizationForceVF,
+ EpilogueTailFoldingCM.value())) {
// we can't apply epilogue TF:
+ LLVM_DEBUG(
+ dbgs() << "LV: Applying epilogue tail-folding failed, disable it.\n");
EpilogueTailFoldingCM.reset();
- IsEpilogueTFEnabled = false;
}
auto [VF, BestPlanPtr] = LVP.computeBestVF();
@@ -8496,7 +8501,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
- ResumeValues, IsEpilogueTFEnabled);
+ ResumeValues, EpilogueTailFoldingCM.has_value());
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 3e6f8add8e035..6b73e5bc6703c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1543,6 +1543,8 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
if (Plan.hasTailFolded()) {
+ assert(!RequiresScalarEpilogue &&
+ "Expected no scalar epilogue for tail-folded plan");
Builder.createNaryOp(VPInstruction::BranchOnCond, Plan.getFalse());
return;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 746e4495e059e..02ce78dcc8e14 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,7 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug -mtriple=aarch64-linux-gnu -mcpu=neoverse-v1 -S %s | FileCheck %s
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug -mcpu=neoverse-v1 -S %s | FileCheck %s
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
+
+target triple = "aarch64-linux-gnu"
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK-LABEL: define void @test_epilogue_tf(
@@ -63,3 +67,30 @@ for.body:
exit:
ret void
}
+
+define i64 @test_no_masked_interleave_support(i64 %y, i32 %n) {
+; CHECK-INVALIDATE-INTERLEAVE-LABEL: Checking a loop in 'test_no_masked_interleave_support'
+; CHECK-INVALIDATE-INTERLEAVE: LV: epilogue tail-folding is enabled
+; CHECK-INVALIDATE-INTERLEAVE: LV: Analyzing interleaved accesses...
+; CHECK-INVALIDATE-INTERLEAVE: LV: Invalidate all interleaved groups due to fold-tail by masking which requires masked-interleaved support
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i32 [ 0, %entry ], [ %inc, %cond.end ]
+ %cmp = icmp eq i64 %y, 0
+ br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+ %div = xor i64 3, %y
+ br label %cond.end
+
+cond.end:
+ %cond = phi i64 [ %div, %cond.false ], [ 77, %for.body ]
+ %inc = add nuw nsw i32 %i, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+ ret i64 %cond
+}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index b56b4c622fd5a..20ede356f2bb2 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -20,6 +20,10 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -vectorize-scev-check-threshold=0 < %s 2>&1 | FileCheck %s \
+; RUN: --check-prefix=CHECK-NO-VPLANS
+
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
@@ -53,6 +57,29 @@ exit:
ret void
}
+; This case can't be tail-folded because all the iterations will be executed by
+; main vector loop.
+define void @test_epilogue_tf_reset(ptr %A) {
+; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf_reset'
+; CHECK: LV: epilogue tail-folding is enabled
+; CHECK: LV: This case of epilogue loop can't be tail-folded.
+; CHECK: LV: Applying epilogue tail-folding failed, disable it.
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, 64
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
; CHECK-LABEL: Checking a loop in 'require_scalar_epilogue'
; CHECK: remark: <unknown>:0:0: Epilogue tail-folding can't be applied because scalar epilogue is required. Fall back to a normal epilogue
@@ -197,6 +224,32 @@ exit:
ret void
}
+; Can't build a valid vplan for this case because too many SCEV checks needed,
+; more than the specfied limit.
+define i64 @test_no_vplan_built(ptr %dst, i64 %n) {
+; CHECK-NO-VPLANS-LABEL: Checking a loop in 'test_no_vplan_built'
+; CHECK-NO-VPLANS: LV: epilogue tail-folding is enabled
+; CHECK-NO-VPLANS: LV: no vplans have been built for main loop VF, bail out of epilogue tail-folding
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %dead.iv = phi i16 [ 0, %entry ], [ %dead.iv.next, %loop ]
+ %prev = phi i64 [ 0, %entry ], [ %ext, %loop ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %dead.iv.next = add i16 %dead.iv, 1
+ %ext = zext i16 %dead.iv.next to i64
+ %gep = getelementptr inbounds i64, ptr %dst, i64 %prev
+ store i64 %iv, ptr %gep, align 8
+ %cmp = icmp slt i64 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %result = phi i64 [ %ext, %loop ]
+ ret i64 %result
+}
+
!1 = distinct !{!1, !2}
!2 = !{!"llvm.loop.vectorize.enable"}
-
>From b4ddbb46e347e85d5ca9a30992699d7c1cfe2785 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 6 Aug 2026 00:43:59 +0100
Subject: [PATCH 05/16] resolve review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 1 -
.../Transforms/Vectorize/LoopVectorize.cpp | 35 ++++++++-----------
.../AArch64/fold-epilogue-tail.ll | 5 +--
.../LoopVectorize/fold-epilogue-tail.ll | 6 ++--
4 files changed, 20 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 7e058d46ef41f..f1bfd70b9c5ee 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -938,7 +938,6 @@ class LoopVectorizationPlanner {
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// tail-folding should be avoided up-front, no plans are generated.
bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC,
- ElementCount EpilogueUserVF,
LoopVectorizationCostModel &EpilogueCM);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ca4cde7caa08e..7cb9e3f3212a1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5514,27 +5514,19 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
}
bool LoopVectorizationPlanner::planForEpilogueTF(
- ElementCount UserVF, unsigned UserIC, ElementCount EpilogueUserVF,
+ ElementCount UserVF, unsigned UserIC,
LoopVectorizationCostModel &EpilogueCM) {
if (VPlans.empty()) {
LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
"out of epilogue tail-folding\n");
return false;
}
- if (!OrigLoop->isInnermost())
- return false;
-
- if (!EpilogueUserVF.isVector() ||
- (estimateElementCount(EpilogueUserVF, Config.getVScaleForTuning()) >=
- estimateElementCount(UserVF, Config.getVScaleForTuning())) *
- UserIC)
- return false;
EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EpilogueCM.computeMaxVF(EpilogueUserVF, /*UserIC*/ 1);
+ EpilogueCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
if (!MaxFactors ||
!EpilogueCM
.foldTailByMasking()) { // Cases that should not to be vectorized
@@ -5545,8 +5537,6 @@ bool LoopVectorizationPlanner::planForEpilogueTF(
}
auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
- if (!VPlan1)
- return false;
if (!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
@@ -5562,20 +5552,23 @@ bool LoopVectorizationPlanner::planForEpilogueTF(
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EpilogueCM.collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ EpilogueCM.collectNonVectorizedAndSetWideningDecisions(
+ EpilogueVectorizationForceVF);
// Expecting only 2 vplans as epilogue TF is applied only for forced VFs.
assert(VPlans.size() == 2 &&
"For tail-folded epilogue, VPlans size is expected to be 2");
// Remove the last vplan, which should be the epilogue plan to replace it by
// the tail-folded vplan:
- assert(VPlans.back()->getSingleVF() == EpilogueUserVF &&
+ assert(VPlans.back()->getSingleVF() == EpilogueVectorizationForceVF &&
"For tail-folded epilogue, last vplan is expected to have "
"EpilogueUserVF");
VPlans.pop_back();
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, EpilogueCM);
+ buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
+ EpilogueVectorizationForceVF, EpilogueCM);
- cost(*VPlans.back(), EpilogueUserVF, /*RU=*/nullptr, EpilogueCM);
+ cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
+ EpilogueCM);
return true;
}
@@ -8214,12 +8207,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- if (EpilogueTailFoldingCM.has_value())
- if (!LVP.planForEpilogueTF(UserVF, UserIC, EpilogueVectorizationForceVF,
- EpilogueTailFoldingCM.value())) {
+ if (EpilogueTailFoldingCM)
+ if (!LVP.planForEpilogueTF(UserVF, UserIC, EpilogueTailFoldingCM.value())) {
// we can't apply epilogue TF:
- LLVM_DEBUG(
- dbgs() << "LV: Applying epilogue tail-folding failed, disable it.\n");
+ reportVectorizationInfo(
+ "Applying epilogue tail-folding failed, disable it.",
+ "InvalidTailFoldedEpilogue", ORE, L);
EpilogueTailFoldingCM.reset();
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 02ce78dcc8e14..e9b248016c32f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,8 +1,9 @@
+; REQUIRES: asserts
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug -mcpu=neoverse-v1 -S %s | FileCheck %s
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug-only=loop-vectorize -mcpu=neoverse-v1 -S %s | FileCheck %s
-; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize,vectorutils --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
target triple = "aarch64-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 20ede356f2bb2..200e599720884 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -20,7 +20,7 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
-; RUN: opt -S -p loop-vectorize -debug --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -vectorize-scev-check-threshold=0 < %s 2>&1 | FileCheck %s \
; RUN: --check-prefix=CHECK-NO-VPLANS
@@ -59,8 +59,8 @@ exit:
; This case can't be tail-folded because all the iterations will be executed by
; main vector loop.
-define void @test_epilogue_tf_reset(ptr %A) {
-; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf_reset'
+define void @test_no_iterations_left(ptr %A) {
+; CHECK-LABEL: LV: Checking a loop in 'test_no_iterations_left'
; CHECK: LV: epilogue tail-folding is enabled
; CHECK: LV: This case of epilogue loop can't be tail-folded.
; CHECK: LV: Applying epilogue tail-folding failed, disable it.
>From 348d4c03d93c428fbbea8121fc50110c72625e34 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Sun, 9 Aug 2026 00:45:24 +0100
Subject: [PATCH 06/16] Swap between default CM instance and EPilogueTF one to
enable costs for tail-folded epilogue
---
.../Vectorize/LoopVectorizationPlanner.h | 44 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 191 ++++++++----------
.../AArch64/fold-epilogue-tail.ll | 2 +-
3 files changed, 111 insertions(+), 126 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f1bfd70b9c5ee..4c1563ce36e0e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -872,8 +872,10 @@ class LoopVectorizationPlanner {
/// The legality analysis.
LoopVectorizationLegality *Legal;
- /// The profitability analysis. Cleared after making cost based decisions.
- std::unique_ptr<LoopVectorizationCostModel> CM;
+ /// The profitability analysis.
+ LoopVectorizationCostModel *EnabledCM;
+ LoopVectorizationCostModel *DefaultCM;
+ LoopVectorizationCostModel *EpilogueTFCM;
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -901,8 +903,7 @@ class LoopVectorizationPlanner {
///
/// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
/// been retired.
- InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
- LoopVectorizationCostModel &EnabledCM) const;
+ InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
/// Precompute costs for certain instructions using the legacy cost model. The
/// function is used to bring up the VPlan-based cost model to initially avoid
@@ -914,21 +915,23 @@ class LoopVectorizationPlanner {
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- std::unique_ptr<LoopVectorizationCostModel> CM,
- VFSelectionContext &Config, InterleavedAccessInfo &IAI,
- PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
+ LoopVectorizationCostModel *DefaultCM,
+ LoopVectorizationCostModel *EpilogueTFCM, VFSelectionContext &Config,
+ InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
+ OptimizationRemarkEmitter *ORE)
+ : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
+ DefaultCM(DefaultCM), EpilogueTFCM(EpilogueTFCM), Config(Config),
+ IAI(IAI), PSE(PSE), ORE(ORE) {
+ enableDefaultCM();
+ }
- ~LoopVectorizationPlanner();
+ void enableDefaultCM() { EnabledCM = DefaultCM; }
- /// Return the cost model. Must not be called after clearCostModel().
- LoopVectorizationCostModel &getCostModel() {
- assert(CM && "Cost model has already been cleared");
- return *CM;
+ void enableEpilogueTFCM() {
+ assert(EpilogueTFCM && "No CM for epilogue tail-folding to enable");
+ EnabledCM = EpilogueTFCM;
}
- /// Destroy the cost model.
- void clearCostModel();
-
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
@@ -937,8 +940,7 @@ class LoopVectorizationPlanner {
/// Build VPlans for the specified \p EpilogueUserVF and \p IC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// tail-folding should be avoided up-front, no plans are generated.
- bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC,
- LoopVectorizationCostModel &EpilogueCM);
+ bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
@@ -1036,7 +1038,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
+ VPlanPtr tryToBuildVPlan1();
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1046,14 +1048,12 @@ class LoopVectorizationPlanner {
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
/// original scalar loop.
- VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
- LoopVectorizationCostModel &EnabledCM);
+ VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
- LoopVectorizationCostModel &EnabledCM);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7cb9e3f3212a1..f1070856c26db 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3136,7 +3136,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
if (VF.isScalar())
continue;
- VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
+ VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
precomputeCosts(*Plan, VF, CostCtx);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
@@ -3572,7 +3572,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Do not interleave tail-folded loops, as the overhead of multiple
// instructions to calculate the predicate is likely not beneficial.
// If an epilogue is not allowed for any other reason, do not interleave.
- if (!CM->isEpilogueAllowed())
+ if (!EnabledCM->isEpilogueAllowed())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
@@ -3606,9 +3606,9 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// then we calculate the cost of VF here.
if (LoopCost == 0) {
if (VF.isScalar())
- LoopCost = CM->expectedCost(VF);
+ LoopCost = EnabledCM->expectedCost(VF);
else
- LoopCost = cost(Plan, VF, &R, *CM);
+ LoopCost = cost(Plan, VF, &R);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
// Loop body is free and there is no need for interleaving.
@@ -3691,7 +3691,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
auto BestKnownTC =
getSmallBestKnownTC(PSE, OrigLoop,
/*CanUseConstantMax=*/true,
- /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
+ /*CanExcludeZeroTrips=*/EnabledCM->isEpilogueAllowed());
// For fixed length VFs treat a scalable trip count as unknown.
if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
@@ -5403,10 +5403,10 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
- CM->collectValuesToIgnore();
- Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
+ EnabledCM->collectValuesToIgnore();
+ Config.collectElementTypesForWidening(&EnabledCM->ValuesToIgnore);
- FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
+ FixedScalableVFPair MaxFactors = EnabledCM->computeMaxVF(UserVF, UserIC);
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
return;
@@ -5417,7 +5417,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1(*CM);
+ auto VPlan1 = tryToBuildVPlan1();
if (!VPlan1)
return;
@@ -5426,7 +5426,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF, *CM);
+ buildVPlans(*VPlan1, VF, VF);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5436,20 +5436,20 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
Config.computeMinimalBitwidths();
// Invalidate interleave groups if all blocks of loop will be predicated.
- if (CM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ if (EnabledCM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs()
<< "LV: Invalidate all interleaved groups due to fold-tail by masking "
"which requires masked-interleaved support.\n");
- if (CM->InterleaveInfo.invalidateGroups())
+ if (EnabledCM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- CM->invalidateCostModelingDecisions();
+ EnabledCM->invalidateCostModelingDecisions();
}
- if (CM->foldTailByMasking())
+ if (EnabledCM->foldTailByMasking())
Legal->prepareToFoldTailByMasking();
ElementCount MaxUserVF =
@@ -5464,23 +5464,23 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
"VF needs to be a power of two");
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(UserVF);
// Build the main-loop VPlan firstly because if epilogue tail-folding is
// enabled, it will be built later, so we keep the epilogue vplans at the
// end.
- buildVPlans(*VPlan1, UserVF, UserVF, *CM);
+ buildVPlans(*VPlan1, UserVF, UserVF);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
- CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
if (UserVF.isScalar() ||
- cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid()) {
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr).isValid()) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
LLVM_DEBUG(printPlans(dbgs()));
return;
@@ -5503,56 +5503,52 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM->collectNonVectorizedAndSetWideningDecisions(VF);
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, *CM);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
- *CM);
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
LLVM_DEBUG(printPlans(dbgs()));
}
-bool LoopVectorizationPlanner::planForEpilogueTF(
- ElementCount UserVF, unsigned UserIC,
- LoopVectorizationCostModel &EpilogueCM) {
+bool LoopVectorizationPlanner::planForEpilogueTF(ElementCount UserVF,
+ unsigned UserIC) {
if (VPlans.empty()) {
LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
"out of epilogue tail-folding\n");
return false;
}
- EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
- EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
+ EnabledCM->ValuesToIgnore.insert_range(DefaultCM->ValuesToIgnore);
+ EnabledCM->VecValuesToIgnore.insert_range(DefaultCM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EpilogueCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
- if (!MaxFactors ||
- !EpilogueCM
- .foldTailByMasking()) { // Cases that should not to be vectorized
- // or tail-folded.
+ EnabledCM->computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
+ if (!MaxFactors || !EnabledCM->foldTailByMasking()) {
+ // Cases that should not to be vectorized // or tail-folded.
reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
return false;
}
- auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
+ auto VPlan1 = tryToBuildVPlan1();
if (!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by "
"masking which requires masked-interleaved support.\n");
- if (EpilogueCM.InterleaveInfo.invalidateGroups())
+ if (EnabledCM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- EpilogueCM.invalidateCostModelingDecisions();
+ EnabledCM->invalidateCostModelingDecisions();
}
Legal->prepareToFoldTailByMasking();
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EpilogueCM.collectNonVectorizedAndSetWideningDecisions(
+ EnabledCM->collectNonVectorizedAndSetWideningDecisions(
EpilogueVectorizationForceVF);
// Expecting only 2 vplans as epilogue TF is applied only for forced VFs.
@@ -5565,10 +5561,9 @@ bool LoopVectorizationPlanner::planForEpilogueTF(
"EpilogueUserVF");
VPlans.pop_back();
buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
- EpilogueVectorizationForceVF, EpilogueCM);
+ EpilogueVectorizationForceVF);
- cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
- EpilogueCM);
+ cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr);
return true;
}
@@ -5764,10 +5759,9 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return Cost;
}
-InstructionCost LoopVectorizationPlanner::cost(
- VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
- LoopVectorizationCostModel &EnabledCM) const {
- VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config,
+InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
+ VPRegisterUsage *RU) const {
+ VPCostContext CostCtx(*TLI, Plan, *EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -5842,7 +5836,7 @@ LoopVectorizationPlanner::computeBestVF() {
"More than a single plan/VF w/o any plan having scalar VF");
// TODO: Compute scalar cost using VPlan-based cost model.
- InstructionCost ScalarCost = CM->expectedCost(ScalarVF);
+ InstructionCost ScalarCost = EnabledCM->expectedCost(ScalarVF);
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
VectorizationFactor BestFactor = ScalarFactor;
@@ -5890,7 +5884,7 @@ LoopVectorizationPlanner::computeBestVF() {
}
InstructionCost Cost =
- cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, *CM);
+ cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
@@ -5913,19 +5907,6 @@ LoopVectorizationPlanner::computeBestVF() {
return {BestFactor, &BestPlan};
}
-LoopVectorizationPlanner::LoopVectorizationPlanner(
- Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
- const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
- InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
- : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
-
-LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
-
-void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
-
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT,
@@ -6541,8 +6522,7 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
- LoopVectorizationCostModel &EnabledCM) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6593,8 +6573,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6625,24 +6605,23 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (EnabledCM.foldTailByMasking())
+ if (EnabledCM->foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
}
-void LoopVectorizationPlanner::buildVPlans(
- VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
- LoopVectorizationCostModel &EnabledCM) {
+void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
+ ElementCount MaxVF) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
- SubRange, EnabledCM);
+ auto Plan =
+ tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()), SubRange);
VF = SubRange.End;
if (!Plan)
@@ -6655,7 +6634,7 @@ void LoopVectorizationPlanner::buildVPlans(
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (EnabledCM.foldTailWithEVL()) {
+ if (EnabledCM->foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6665,7 +6644,7 @@ void LoopVectorizationPlanner::buildVPlans(
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6676,8 +6655,8 @@ void LoopVectorizationPlanner::buildVPlans(
}
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
- VPlanPtr Plan, VFRange &Range, LoopVectorizationCostModel &EnabledCM) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
+ VFRange &Range) {
// For outer loops, the plan only needs basic recipe conversion and induction
// live-out optimization; the full inner-loop recipe building below does not
@@ -6703,8 +6682,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
- [&EnabledCM](ElementCount VF) {
- return !EnabledCM.requiresScalarEpilogue(VF.isVector());
+ [&](ElementCount VF) {
+ return !EnabledCM->requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6722,9 +6701,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&EnabledCM, VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(EnabledCM, VF);
- TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6751,10 +6730,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
for (InterleaveGroup<Instruction> *IG :
- EnabledCM.InterleaveInfo.getInterleaveGroups()) {
- auto ApplyIG = [IG, &EnabledCM](ElementCount VF) -> bool {
+ EnabledCM->InterleaveInfo.getInterleaveGroups()) {
+ auto ApplyIG = [IG, this](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- EnabledCM.getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM->getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6771,7 +6750,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, EnabledCM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, *EnabledCM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6782,7 +6761,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, EnabledCM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6889,7 +6868,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, EnabledCM.isEpilogueAllowed());
+ InterleaveGroups, EnabledCM->isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6906,7 +6885,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (EnabledCM.maskPartialAliasing())
+ if (EnabledCM->maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -6950,7 +6929,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// Remove the predicated select if the target doesn't want it.
VPValue *V;
- if (!CM->usePredicatedReductionSelect(RecurrenceKind) &&
+ if (!EnabledCM->usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
@@ -8173,15 +8152,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- // Use the planner for vectorization.
- LoopVectorizationPlanner LVP(
- L, LI, DT, TLI, *TTI, &LVL,
- std::make_unique<LoopVectorizationCostModel>(
- SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
- Config, IAI, PSE, ORE);
+ LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
+ GetBFI, F, IAI, Config);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
+ getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
if (EpilogueTailLoweringStatus ==
@@ -8195,6 +8170,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
*TailFoldingCMIAI, Config);
}
+ // Use the planner for vectorization.
+ LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, &CM,
+ EpilogueTailFoldingCM ? &*EpilogueTailFoldingCM
+ : nullptr,
+ Config, IAI, PSE, ORE);
+
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
@@ -8207,14 +8188,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- if (EpilogueTailFoldingCM)
- if (!LVP.planForEpilogueTF(UserVF, UserIC, EpilogueTailFoldingCM.value())) {
+ if (EpilogueTailFoldingCM) {
+ // Enable the epilogue tail-folding CM
+ LVP.enableEpilogueTFCM();
+ if (!LVP.planForEpilogueTF(UserVF, UserIC)) {
// we can't apply epilogue TF:
reportVectorizationInfo(
"Applying epilogue tail-folding failed, disable it.",
"InvalidTailFoldedEpilogue", ORE, L);
EpilogueTailFoldingCM.reset();
}
+ // Get back the default CM:
+ LVP.enableDefaultCM();
+ }
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
@@ -8227,11 +8213,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
LVP.emitInvalidCostRemarks(ORE);
- assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
+ assert((IsInnerLoop || !CM.maskPartialAliasing()) &&
"Did not expect to alias-mask outer loop");
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
- LVP.getCostModel().maskPartialAliasing());
+ CM.maskPartialAliasing());
if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8257,7 +8243,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
+ VPCostContext CostCtx(*TLI, *BestPlanPtr, CM, Config,
/*ReusePrintingSlotTracker=*/true);
if (!ForceVectorization &&
!isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
@@ -8340,7 +8326,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
- if (LVP.getCostModel().maskPartialAliasing()) {
+ if (CM.maskPartialAliasing()) {
LLVM_DEBUG(
dbgs()
<< "LV: Not interleaving due to partial aliasing vectorization.\n");
@@ -8420,11 +8406,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Whether a scalar epilogue may be created is decided by the epilogue
// lowering policy.
// TODO: Also move check to be based on VPlan.
- bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
-
- // Destroy the cost model before executing any plan, so that code generation
- // cannot rely on cost-modeling decisions.
- LVP.clearCostModel();
+ bool ScalarEpilogueAllowed = CM.isEpilogueAllowed();
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
@@ -8464,6 +8446,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
+ if (EpilogueTailFoldingCM)
+ LVP.enableEpilogueTFCM();
// Derive EPI fields from VPlan-generated IR.
BasicBlock *EntryBB =
cast<VPIRBasicBlock>(BestMainPlan.getEntry())->getIRBasicBlock();
@@ -8496,6 +8480,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
ResumeValues, EpilogueTailFoldingCM.has_value());
++LoopsEpilogueVectorized;
+ LVP.enableDefaultCM();
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
BestPlan);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index e9b248016c32f..ece1fc3a15b11 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,5 +1,5 @@
-; REQUIRES: asserts
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; REQUIRES: asserts
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug-only=loop-vectorize -mcpu=neoverse-v1 -S %s | FileCheck %s
>From 232648edcc1027b4c70a701085268a5767f68e9f Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 10 Aug 2026 21:03:58 +0100
Subject: [PATCH 07/16] resolve review comments - improve readability
---
.../Vectorize/LoopVectorizationPlanner.h | 15 +++++++++++----
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 5 ++---
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4c1563ce36e0e..03d3b8121ed0b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -873,8 +873,16 @@ class LoopVectorizationPlanner {
LoopVectorizationLegality *Legal;
/// The profitability analysis.
+ /// The CM currently in effect for the VPlan being built or costed; it always
+ /// aliases either \c DefaultCM or \c EpilogueTFCM.
LoopVectorizationCostModel *EnabledCM;
+ /// The CM used for the main-loop VPlan, and for the epilogue VPlan in all
+ /// cases except tail-folded epilogue vectorization.
LoopVectorizationCostModel *DefaultCM;
+ /// The CM used only when the epilogue loop is vectorized with tail-folding.
+ /// \c EnabledCM is switched to point here (via enableEpilogueTFCM()) while
+ /// the epilogue VPlan's costs are computed, so that they correctly account
+ /// for the tail-folded epilogue.
LoopVectorizationCostModel *EpilogueTFCM;
/// VF selection state independent of cost-modeling decisions.
@@ -937,10 +945,9 @@ class LoopVectorizationPlanner {
/// interleaving should be avoided up-front, no plans are generated.
void plan(ElementCount UserVF, unsigned UserIC);
- /// Build VPlans for the specified \p EpilogueUserVF and \p IC if they are
- /// non-zero or all applicable candidate VFs otherwise. If vectorization and
- /// tail-folding should be avoided up-front, no plans are generated.
- bool planForEpilogueTF(ElementCount UserVF, unsigned UserIC);
+ /// Build VPlan for the forced epilogue VF. If vectorization and tail-folding
+ /// should be avoided up-front, no tail-folded plans are generated.
+ bool planForEpilogueTF();
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f1070856c26db..95959781a8276 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5512,8 +5512,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
LLVM_DEBUG(printPlans(dbgs()));
}
-bool LoopVectorizationPlanner::planForEpilogueTF(ElementCount UserVF,
- unsigned UserIC) {
+bool LoopVectorizationPlanner::planForEpilogueTF() {
if (VPlans.empty()) {
LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
"out of epilogue tail-folding\n");
@@ -8191,7 +8190,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (EpilogueTailFoldingCM) {
// Enable the epilogue tail-folding CM
LVP.enableEpilogueTFCM();
- if (!LVP.planForEpilogueTF(UserVF, UserIC)) {
+ if (!LVP.planForEpilogueTF()) {
// we can't apply epilogue TF:
reportVectorizationInfo(
"Applying epilogue tail-folding failed, disable it.",
>From c0daebf2e6f9bf811f998b4aecd75e8623b96438 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 11 Aug 2026 15:59:31 +0100
Subject: [PATCH 08/16] disallow epilogue TF for outer loop
---
.../LoopVectorize/fold-epilogue-tail.ll | 59 ++++++++++---------
1 file changed, 31 insertions(+), 28 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 200e599720884..e96850b65316c 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -83,6 +83,7 @@ exit:
define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
; CHECK-LABEL: Checking a loop in 'require_scalar_epilogue'
; CHECK: remark: <unknown>:0:0: Epilogue tail-folding can't be applied because scalar epilogue is required. Fall back to a normal epilogue
+; CHECK-NOT: LV: epilogue tail-folding is enabled
;
entry:
br label %loop.header
@@ -111,6 +112,7 @@ exit.2:
define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
; CHECK-LABEL: Checking a loop in 'opt_for_size'
; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue is allowed
+; CHECK-NOT: LV: epilogue tail-folding is enabled
;
entry:
br label %for.body
@@ -196,34 +198,6 @@ exit:
ret void
}
-define void @test_outer_loop(ptr %A, i64 %m) {
-; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
-; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
-;
-entry:
- br label %outer.header
-
-outer.header:
- %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
- br label %inner
-
-inner:
- %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
- %gep = getelementptr inbounds i8, ptr %A, i64 %iv.inner
- store i8 0, ptr %gep, align 1
- %iv.inner.next = add nuw nsw i64 %iv.inner, 1
- %inner.ec = icmp eq i64 %iv.inner.next, 8
- br i1 %inner.ec, label %outer.latch, label %inner
-
-outer.latch:
- %iv.outer.next = add nuw nsw i64 %iv.outer, 1
- %outer.ec = icmp eq i64 %iv.outer.next, %m
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
; Can't build a valid vplan for this case because too many SCEV checks needed,
; more than the specfied limit.
define i64 @test_no_vplan_built(ptr %dst, i64 %n) {
@@ -251,5 +225,34 @@ exit:
ret i64 %result
}
+define void @test_outer_loop(ptr %A, i64 %m) {
+; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
+; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
+; CHECK-OUTER-LOOP-NOT: LV: epilogue tail-folding is enabled
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv.inner
+ store i32 0, ptr %gep, align 4
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.ec = icmp eq i64 %iv.inner.next, 8
+ br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+ %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+ %outer.ec = icmp eq i64 %iv.outer.next, %m
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+ ret void
+}
+
!1 = distinct !{!1, !2}
!2 = !{!"llvm.loop.vectorize.enable"}
>From 79e4287a649abb9dd2def4c5af54b3f02b75464f Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 14 Aug 2026 16:57:33 +0100
Subject: [PATCH 09/16] revert commit of swapping between different CMs
---
.../Vectorize/LoopVectorizationPlanner.h | 51 ++---
.../Transforms/Vectorize/LoopVectorize.cpp | 189 ++++++++++--------
2 files changed, 123 insertions(+), 117 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 03d3b8121ed0b..e3ff60b29817d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -872,18 +872,8 @@ class LoopVectorizationPlanner {
/// The legality analysis.
LoopVectorizationLegality *Legal;
- /// The profitability analysis.
- /// The CM currently in effect for the VPlan being built or costed; it always
- /// aliases either \c DefaultCM or \c EpilogueTFCM.
- LoopVectorizationCostModel *EnabledCM;
- /// The CM used for the main-loop VPlan, and for the epilogue VPlan in all
- /// cases except tail-folded epilogue vectorization.
- LoopVectorizationCostModel *DefaultCM;
- /// The CM used only when the epilogue loop is vectorized with tail-folding.
- /// \c EnabledCM is switched to point here (via enableEpilogueTFCM()) while
- /// the epilogue VPlan's costs are computed, so that they correctly account
- /// for the tail-folded epilogue.
- LoopVectorizationCostModel *EpilogueTFCM;
+ /// The profitability analysis. Cleared after making cost based decisions.
+ std::unique_ptr<LoopVectorizationCostModel> CM;
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -911,7 +901,8 @@ class LoopVectorizationPlanner {
///
/// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
/// been retired.
- InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
+ InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const;
/// Precompute costs for certain instructions using the legacy cost model. The
/// function is used to bring up the VPlan-based cost model to initially avoid
@@ -923,23 +914,21 @@ class LoopVectorizationPlanner {
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel *DefaultCM,
- LoopVectorizationCostModel *EpilogueTFCM, VFSelectionContext &Config,
- InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
- : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- DefaultCM(DefaultCM), EpilogueTFCM(EpilogueTFCM), Config(Config),
- IAI(IAI), PSE(PSE), ORE(ORE) {
- enableDefaultCM();
- }
+ std::unique_ptr<LoopVectorizationCostModel> CM,
+ VFSelectionContext &Config, InterleavedAccessInfo &IAI,
+ PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
- void enableDefaultCM() { EnabledCM = DefaultCM; }
+ ~LoopVectorizationPlanner();
- void enableEpilogueTFCM() {
- assert(EpilogueTFCM && "No CM for epilogue tail-folding to enable");
- EnabledCM = EpilogueTFCM;
+ /// Return the cost model. Must not be called after clearCostModel().
+ LoopVectorizationCostModel &getCostModel() {
+ assert(CM && "Cost model has already been cleared");
+ return *CM;
}
+ /// Destroy the cost model.
+ void clearCostModel();
+
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
@@ -947,7 +936,7 @@ class LoopVectorizationPlanner {
/// Build VPlan for the forced epilogue VF. If vectorization and tail-folding
/// should be avoided up-front, no tail-folded plans are generated.
- bool planForEpilogueTF();
+ bool planForEpilogueTF(LoopVectorizationCostModel &EpilogueCM);
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
@@ -1045,7 +1034,7 @@ class LoopVectorizationPlanner {
/// Build an initial VPlan, with HCFG wrapping the original scalar loop and
/// scalar transformations applied. Returns null if an initial VPlan cannot
/// be built.
- VPlanPtr tryToBuildVPlan1();
+ VPlanPtr tryToBuildVPlan1(LoopVectorizationCostModel &EnabledCM);
/// Build a VPlan using VPRecipes according to the information gathered by
/// Legal and VPlan-based analysis. For outer loops, performs basic recipe
@@ -1055,12 +1044,14 @@ class LoopVectorizationPlanner {
/// maximum VF for which no plan could be built. Each VPlan is built starting
/// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
/// original scalar loop.
- VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
+ VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range,
+ LoopVectorizationCostModel &EnabledCM);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// based on \p VPlan1 and according to the information gathered by Legal
/// when it checked if it is legal to vectorize the loop.
- void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
+ void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM);
/// Add ComputeReductionResult recipes to the middle block to compute the
/// final reduction results. Add Select recipes to the latch block when
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 95959781a8276..a7cb92fcc4e8b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3136,7 +3136,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
if (VF.isScalar())
continue;
- VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config,
+ VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
/*ReusePrintingSlotTracker=*/true);
precomputeCosts(*Plan, VF, CostCtx);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
@@ -3572,7 +3572,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Do not interleave tail-folded loops, as the overhead of multiple
// instructions to calculate the predicate is likely not beneficial.
// If an epilogue is not allowed for any other reason, do not interleave.
- if (!EnabledCM->isEpilogueAllowed())
+ if (!CM->isEpilogueAllowed())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
@@ -3606,9 +3606,9 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// then we calculate the cost of VF here.
if (LoopCost == 0) {
if (VF.isScalar())
- LoopCost = EnabledCM->expectedCost(VF);
+ LoopCost = CM->expectedCost(VF);
else
- LoopCost = cost(Plan, VF, &R);
+ LoopCost = cost(Plan, VF, &R, *CM);
assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
// Loop body is free and there is no need for interleaving.
@@ -3691,7 +3691,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
auto BestKnownTC =
getSmallBestKnownTC(PSE, OrigLoop,
/*CanUseConstantMax=*/true,
- /*CanExcludeZeroTrips=*/EnabledCM->isEpilogueAllowed());
+ /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
// For fixed length VFs treat a scalable trip count as unknown.
if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
@@ -5403,10 +5403,10 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
}
void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
- EnabledCM->collectValuesToIgnore();
- Config.collectElementTypesForWidening(&EnabledCM->ValuesToIgnore);
+ CM->collectValuesToIgnore();
+ Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
- FixedScalableVFPair MaxFactors = EnabledCM->computeMaxVF(UserVF, UserIC);
+ FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
return;
@@ -5417,7 +5417,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
Legal->collectUnitStridePredicates();
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(*CM);
if (!VPlan1)
return;
@@ -5426,7 +5426,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// plan for that VF only.
ElementCount VF =
MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
- buildVPlans(*VPlan1, VF, VF);
+ buildVPlans(*VPlan1, VF, VF, *CM);
LLVM_DEBUG(printPlans(dbgs()));
return;
}
@@ -5436,20 +5436,20 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
Config.computeMinimalBitwidths();
// Invalidate interleave groups if all blocks of loop will be predicated.
- if (EnabledCM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
+ if (CM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs()
<< "LV: Invalidate all interleaved groups due to fold-tail by masking "
"which requires masked-interleaved support.\n");
- if (EnabledCM->InterleaveInfo.invalidateGroups())
+ if (CM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- EnabledCM->invalidateCostModelingDecisions();
+ CM->invalidateCostModelingDecisions();
}
- if (EnabledCM->foldTailByMasking())
+ if (CM->foldTailByMasking())
Legal->prepareToFoldTailByMasking();
ElementCount MaxUserVF =
@@ -5464,23 +5464,23 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
"VF needs to be a power of two");
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EnabledCM->collectNonVectorizedAndSetWideningDecisions(UserVF);
+ CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
// Build the main-loop VPlan firstly because if epilogue tail-folding is
// enabled, it will be built later, so we keep the epilogue vplans at the
// end.
- buildVPlans(*VPlan1, UserVF, UserVF);
+ buildVPlans(*VPlan1, UserVF, UserVF, *CM);
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
- EnabledCM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF);
+ CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
}
if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
if (UserVF.isScalar() ||
- cost(*VPlans.front(), UserVF, /*RU=*/nullptr).isValid()) {
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid()) {
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
LLVM_DEBUG(printPlans(dbgs()));
return;
@@ -5503,51 +5503,55 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- EnabledCM->collectNonVectorizedAndSetWideningDecisions(VF);
+ CM->collectNonVectorizedAndSetWideningDecisions(VF);
}
- buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF);
- buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF);
+ buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, *CM);
+ buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
+ *CM);
LLVM_DEBUG(printPlans(dbgs()));
}
-bool LoopVectorizationPlanner::planForEpilogueTF() {
+bool LoopVectorizationPlanner::planForEpilogueTF(
+ LoopVectorizationCostModel &EpilogueCM) {
if (VPlans.empty()) {
LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
"out of epilogue tail-folding\n");
return false;
}
- EnabledCM->ValuesToIgnore.insert_range(DefaultCM->ValuesToIgnore);
- EnabledCM->VecValuesToIgnore.insert_range(DefaultCM->VecValuesToIgnore);
+ EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
+ EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EnabledCM->computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
- if (!MaxFactors || !EnabledCM->foldTailByMasking()) {
- // Cases that should not to be vectorized // or tail-folded.
+ EpilogueCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
+ if (!MaxFactors ||
+ !EpilogueCM
+ .foldTailByMasking()) { // Cases that should not to be vectorized
+ // or tail-folded.
reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
return false;
}
- auto VPlan1 = tryToBuildVPlan1();
+ auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
if (!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by "
"masking which requires masked-interleaved support.\n");
- if (EnabledCM->InterleaveInfo.invalidateGroups())
+ if (EpilogueCM.InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- EnabledCM->invalidateCostModelingDecisions();
+ EpilogueCM.invalidateCostModelingDecisions();
}
Legal->prepareToFoldTailByMasking();
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EnabledCM->collectNonVectorizedAndSetWideningDecisions(
+ EpilogueCM.collectNonVectorizedAndSetWideningDecisions(
EpilogueVectorizationForceVF);
// Expecting only 2 vplans as epilogue TF is applied only for forced VFs.
@@ -5560,9 +5564,10 @@ bool LoopVectorizationPlanner::planForEpilogueTF() {
"EpilogueUserVF");
VPlans.pop_back();
buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
- EpilogueVectorizationForceVF);
+ EpilogueVectorizationForceVF, EpilogueCM);
- cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr);
+ cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
+ EpilogueCM);
return true;
}
@@ -5758,9 +5763,10 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return Cost;
}
-InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
- VPRegisterUsage *RU) const {
- VPCostContext CostCtx(*TLI, Plan, *EnabledCM, Config,
+InstructionCost LoopVectorizationPlanner::cost(
+ VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const {
+ VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -5835,7 +5841,7 @@ LoopVectorizationPlanner::computeBestVF() {
"More than a single plan/VF w/o any plan having scalar VF");
// TODO: Compute scalar cost using VPlan-based cost model.
- InstructionCost ScalarCost = EnabledCM->expectedCost(ScalarVF);
+ InstructionCost ScalarCost = CM->expectedCost(ScalarVF);
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
VectorizationFactor BestFactor = ScalarFactor;
@@ -5883,7 +5889,7 @@ LoopVectorizationPlanner::computeBestVF() {
}
InstructionCost Cost =
- cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
+ cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, *CM);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
@@ -5906,6 +5912,19 @@ LoopVectorizationPlanner::computeBestVF() {
return {BestFactor, &BestPlan};
}
+LoopVectorizationPlanner::LoopVectorizationPlanner(
+ Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
+ std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
+ InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
+ OptimizationRemarkEmitter *ORE)
+ : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
+ CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+
+LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
+
+void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
+
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT,
@@ -6521,7 +6540,8 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1(
+ LoopVectorizationCostModel &EnabledCM) {
bool IsInnerLoop = OrigLoop->isInnermost();
// Set up loop versioning for inner loops with memory runtime checks.
@@ -6572,8 +6592,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
!ForceVectorization &&
- (EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- EnabledCM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ (EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
+ EnabledCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
@@ -6604,23 +6624,24 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
- if (EnabledCM->foldTailByMasking())
+ if (EnabledCM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
}
-void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
- ElementCount MaxVF) {
+void LoopVectorizationPlanner::buildVPlans(
+ VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF,
+ LoopVectorizationCostModel &EnabledCM) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
return;
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
VFRange SubRange = {VF, MaxVFTimes2};
- auto Plan =
- tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()), SubRange);
+ auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
+ SubRange, EnabledCM);
VF = SubRange.End;
if (!Plan)
@@ -6633,7 +6654,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
- if (EnabledCM->foldTailWithEVL()) {
+ if (EnabledCM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
@@ -6643,7 +6664,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
- TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
@@ -6654,8 +6675,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
}
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
- VFRange &Range) {
+VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(
+ VPlanPtr Plan, VFRange &Range, LoopVectorizationCostModel &EnabledCM) {
// For outer loops, the plan only needs basic recipe conversion and induction
// live-out optimization; the full inner-loop recipe building below does not
@@ -6681,8 +6702,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool RequiresScalarEpilogueCheck =
LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) {
- return !EnabledCM->requiresScalarEpilogue(VF.isVector());
+ [&EnabledCM](ElementCount VF) {
+ return !EnabledCM.requiresScalarEpilogue(VF.isVector());
},
Range);
// Update the branch in the middle block if a scalar epilogue is required.
@@ -6700,9 +6721,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
- IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(EnabledCM, VF);
+ IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&EnabledCM, VF);
- TailFoldingStyle Style = EnabledCM->getTailFoldingStyle();
+ TailFoldingStyle Style = EnabledCM.getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -6729,10 +6750,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
for (InterleaveGroup<Instruction> *IG :
- EnabledCM->InterleaveInfo.getInterleaveGroups()) {
- auto ApplyIG = [IG, this](ElementCount VF) -> bool {
+ EnabledCM.InterleaveInfo.getInterleaveGroups()) {
+ auto ApplyIG = [IG, &EnabledCM](ElementCount VF) -> bool {
bool Result = (VF.isVector() && // Query is illegal for VF == 1
- EnabledCM->getWideningDecision(IG->getInsertPos(), VF) ==
+ EnabledCM.getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
// For scalable vectors, the interleave factors must be <= 8 since we
// require the (de)interleaveN intrinsics instead of shufflevectors.
@@ -6749,7 +6770,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, Legal, *EnabledCM, Builder);
+ VPRecipeBuilder RecipeBuilder(*Plan, Legal, EnabledCM, Builder);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -6760,7 +6781,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
- VPCostContext CostCtx(*TLI, *Plan, *EnabledCM, Config);
+ VPCostContext CostCtx(*TLI, *Plan, EnabledCM, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
@@ -6867,7 +6888,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, EnabledCM->isEpilogueAllowed());
+ InterleaveGroups, EnabledCM.isEpilogueAllowed());
// Convert memory recipes to strided access recipes if the strided access is
// legal and profitable.
@@ -6884,7 +6905,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (EnabledCM->maskPartialAliasing())
+ if (EnabledCM.maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
@@ -6928,7 +6949,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// Remove the predicated select if the target doesn't want it.
VPValue *V;
- if (!EnabledCM->usePredicatedReductionSelect(RecurrenceKind) &&
+ if (!CM->usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
@@ -8151,11 +8172,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
- GetBFI, F, IAI, Config);
+ // Use the planner for vectorization.
+ LoopVectorizationPlanner LVP(
+ L, LI, DT, TLI, *TTI, &LVL,
+ std::make_unique<LoopVectorizationCostModel>(
+ SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
+ Config, IAI, PSE, ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
+ getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
if (EpilogueTailLoweringStatus ==
@@ -8169,12 +8194,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
*TailFoldingCMIAI, Config);
}
- // Use the planner for vectorization.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, &CM,
- EpilogueTailFoldingCM ? &*EpilogueTailFoldingCM
- : nullptr,
- Config, IAI, PSE, ORE);
-
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
@@ -8187,19 +8206,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- if (EpilogueTailFoldingCM) {
- // Enable the epilogue tail-folding CM
- LVP.enableEpilogueTFCM();
- if (!LVP.planForEpilogueTF()) {
+ if (EpilogueTailFoldingCM)
+ if (!LVP.planForEpilogueTF(EpilogueTailFoldingCM.value())) {
// we can't apply epilogue TF:
reportVectorizationInfo(
"Applying epilogue tail-folding failed, disable it.",
"InvalidTailFoldedEpilogue", ORE, L);
EpilogueTailFoldingCM.reset();
}
- // Get back the default CM:
- LVP.enableDefaultCM();
- }
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
@@ -8212,11 +8226,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
LVP.emitInvalidCostRemarks(ORE);
- assert((IsInnerLoop || !CM.maskPartialAliasing()) &&
+ assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
"Did not expect to alias-mask outer loop");
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
- CM.maskPartialAliasing());
+ LVP.getCostModel().maskPartialAliasing());
if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8242,7 +8256,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- VPCostContext CostCtx(*TLI, *BestPlanPtr, CM, Config,
+ VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
/*ReusePrintingSlotTracker=*/true);
if (!ForceVectorization &&
!isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
@@ -8325,7 +8339,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
- if (CM.maskPartialAliasing()) {
+ if (LVP.getCostModel().maskPartialAliasing()) {
LLVM_DEBUG(
dbgs()
<< "LV: Not interleaving due to partial aliasing vectorization.\n");
@@ -8405,7 +8419,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Whether a scalar epilogue may be created is decided by the epilogue
// lowering policy.
// TODO: Also move check to be based on VPlan.
- bool ScalarEpilogueAllowed = CM.isEpilogueAllowed();
+ bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
+
+ // Destroy the cost model before executing any plan, so that code generation
+ // cannot rely on cost-modeling decisions.
+ LVP.clearCostModel();
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
@@ -8445,8 +8463,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
- if (EpilogueTailFoldingCM)
- LVP.enableEpilogueTFCM();
// Derive EPI fields from VPlan-generated IR.
BasicBlock *EntryBB =
cast<VPIRBasicBlock>(BestMainPlan.getEntry())->getIRBasicBlock();
@@ -8479,7 +8495,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
ResumeValues, EpilogueTailFoldingCM.has_value());
++LoopsEpilogueVectorized;
- LVP.enableDefaultCM();
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
BestPlan);
>From e3c31b92aba933d75f796b6865516ee2e72ab473 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 19 Aug 2026 16:06:42 +0100
Subject: [PATCH 10/16] Add test cases for different scenarios that should be
supported by the feature
---
.../AArch64/fold-epilogue-tail.ll | 412 +++++++++++++++++-
1 file changed, 399 insertions(+), 13 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index ece1fc3a15b11..79495c4ad9c3e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -8,9 +8,9 @@
target triple = "aarch64-linux-gnu"
-define void @test_epilogue_tf(ptr %A, i64 %n) {
+define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-LABEL: define void @test_epilogue_tf(
-; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
@@ -20,13 +20,15 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
-; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP1]], align 1
-; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 16
+; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4
+; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -38,14 +40,16 @@ define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <8 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT2]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX2]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> splat (i8 1), ptr align 1 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], 8
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX4]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[BROADCAST_SPLAT3]], ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT5]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT5]], i64 [[N]])
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -59,8 +63,8 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
- store i8 1, ptr %arrayidx, align 1
+ %arrayidx = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i32 %val, ptr %arrayidx, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp ne i64 %iv.next, %n
br i1 %exitcond, label %for.body, label %exit
@@ -69,6 +73,388 @@ exit:
ret void
}
+define i32 @add_redc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @add_redc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4]] = add <16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[TMP8]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP9]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = add <8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
+; CHECK-NEXT: [[TMP11]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP10]], <8 x i32> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %add = add i32 %load, %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %add
+}
+
+define i32 @max_redc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @max_redc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD]], <16 x i32> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD3]], <16 x i32> [[VEC_PHI2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[TMP4]], <16 x i32> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> [[RDX_MINMAX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP8]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[WIDE_MASKED_LOAD]], <8 x i32> [[VEC_PHI5]])
+; CHECK-NEXT: [[TMP10]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP9]], <8 x i32> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> [[TMP10]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[MAX_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %max, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %max = call i32 @llvm.umax(i32 %load, i32 %red)
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %max
+}
+
+define i32 @live-out(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @live-out(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[WIDE_LOAD]], i64 15
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP5]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = xor <8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP8]], i1 false)
+; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[FOR_END]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[LOAD_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds nuw i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %for.end, label %loop
+
+for.end:
+ ret i32 %load
+}
+
+define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
+; CHECK-LABEL: define void @reversed-loop(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[ST:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2
+; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 -1)
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[SMIN1]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[N]], -2
+; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP3]], i32 -1)
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], [[SMIN]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[ST]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP5]], [[ST]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i32 [[TMP2]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[TMP2]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP7]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[ST]], [[N_VEC]]
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLAT]], <16 x i32> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[ST]], [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 -15
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 -31
+; CHECK-NEXT: store <16 x i32> [[REVERSE]], ptr [[TMP11]], align 4
+; CHECK-NEXT: store <16 x i32> [[REVERSE]], ptr [[TMP12]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLAT4]], <8 x i32> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[VEC_EPILOG_RESUME_VAL]], i32 [[TMP2]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = sub i32 [[ST]], [[INDEX6]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[TMP15]], i64 -7
+; CHECK-NEXT: [[REVERSE7:%.*]] = shufflevector <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[REVERSE5]], ptr align 4 [[TMP16]], <8 x i1> [[REVERSE7]])
+; CHECK-NEXT: [[INDEX_NEXT8]] = add i32 [[INDEX6]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT8]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[ST]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
+; CHECK-NEXT: store i32 [[VAL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = sub nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sge i32 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %st = sub i32 %n, 1
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %st, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %A, i32 %iv
+ store i32 %val, ptr %arrayidx, align 4
+ %iv.next = sub nuw nsw i32 %iv, 1
+ %exitcond = icmp sge i32 %iv.next, 0
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
+define void @math_func(ptr %A, i32 %n) {
+; CHECK-LABEL: define void @math_func(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[UMAX]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i32 [[UMAX]], 15
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[UMAX]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x float> @llvm.pow.v16f32(<16 x float> [[WIDE_LOAD]], <16 x float> splat (float 2.000000e+00))
+; CHECK-NEXT: store <16 x float> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[VEC_EPILOG_RESUME_VAL]], i32 [[UMAX]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i32 [[INDEX2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x float> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = call <8 x float> @llvm.pow.v8f32(<8 x float> [[WIDE_MASKED_LOAD]], <8 x float> splat (float 2.000000e+00))
+; CHECK-NEXT: call void @llvm.masked.store.v8f32.p0(<8 x float> [[TMP5]], ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT3]] = add i32 [[INDEX2]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT3]], i32 [[UMAX]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %A, i32 %iv
+ %load = load float, ptr %arrayidx, align 4
+ %val = call float @llvm.pow.f32(float %load, float 2.0)
+ store float %val, ptr %arrayidx, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp ult i32 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
define i64 @test_no_masked_interleave_support(i64 %y, i32 %n) {
; CHECK-INVALIDATE-INTERLEAVE-LABEL: Checking a loop in 'test_no_masked_interleave_support'
; CHECK-INVALIDATE-INTERLEAVE: LV: epilogue tail-folding is enabled
>From 30b7b22f7c763f451f04d5d86c399c08905d9fbe Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 24 Aug 2026 10:31:47 +0100
Subject: [PATCH 11/16] Give Planner its own epilogue tail-folding CM and make
plan() responsible for handling epilogueTF planning
---
.../Vectorize/LoopVectorizationPlanner.h | 10 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 344 ++++++++------
.../AArch64/fold-epilogue-tail.ll | 435 ++++++++++++++----
3 files changed, 558 insertions(+), 231 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index e3ff60b29817d..3399d795b04fd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -875,6 +875,10 @@ class LoopVectorizationPlanner {
/// The profitability analysis. Cleared after making cost based decisions.
std::unique_ptr<LoopVectorizationCostModel> CM;
+ /// The profitability analysis for epilogue tail-folding.
+ /// Cleared after making cost based decisions.
+ std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM;
+
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -915,6 +919,7 @@ class LoopVectorizationPlanner {
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM,
+ std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM,
VFSelectionContext &Config, InterleavedAccessInfo &IAI,
PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
@@ -929,6 +934,9 @@ class LoopVectorizationPlanner {
/// Destroy the cost model.
void clearCostModel();
+ /// Destroy the epilogue tail-folding cost model.
+ void clearEpilogueTfCM();
+
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
@@ -936,7 +944,7 @@ class LoopVectorizationPlanner {
/// Build VPlan for the forced epilogue VF. If vectorization and tail-folding
/// should be avoided up-front, no tail-folded plans are generated.
- bool planForEpilogueTF(LoopVectorizationCostModel &EpilogueCM);
+ bool planForEpilogueTF();
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a7cb92fcc4e8b..26b31b8c184d0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3011,6 +3011,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
+ // TODO: Make NoScalarEpilogueNeeded lambda a separate function to be used
+ // only for main loop VF not also epilogueVF. Using it for epilogueVF against
+ // full TC is inaccurate.
auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
@@ -3128,9 +3131,13 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
+ // Skip cost remarks when Plan is not compatible with the CM.
+ // Specifically for the case of epilogue tail-folded Plans.
+ if (Plan->hasTailFolded() ^ CM->preferTailFoldedLoop())
+ continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
- // Querying VPlan-based cost model with a scarlar VF will cause some
+ // Querying VPlan-based cost model with a scalar VF will cause some
// errors because we expect the VF is vector for most of the widen
// recipes.
if (VF.isScalar())
@@ -3352,6 +3359,78 @@ static bool hasFindLastReductionPhi(VPlan &Plan) {
RedPhi->getRecurrenceKind());
});
}
+/// Determine how to lower the epilogue for the vector epilogue loop.
+/// Check if there are any conflicts that prevent tail-folding the epilogue.
+/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
+/// otherwise CM_EpilogueAllowed.
+static EpilogueLowering
+getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
+ OptimizationRemarkEmitter *ORE,
+ const LoopVectorizationLegality &LVL,
+ const LoopVectorizeHints &Hints) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
+ EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
+ return CM_EpilogueAllowed;
+
+ if (!EnableEpilogueVectorization) {
+ reportVectorizationInfo(
+ "Options conflict, epilogue vectorization is disallowed while "
+ "epilogue tail-folding allowed!",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
+ reportVectorizationInfo("For now, epilogue tail-folding can't be "
+ "applied without forced main/epilogue loop VF",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (ElementCount::isKnownLE(Hints.getWidth(), EpilogueVectorizationForceVF)) {
+ reportVectorizationInfo("For now, epilogue tail-folding can't be applied "
+ "when VF of the main loop <= VF of the epilogue",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!L->isInnermost()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported for outer loop",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding can't be applied because scalar epilogue is "
+ "required. Fall back to a normal epilogue",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!MainCM.isEpilogueAllowed()) {
+ reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
+ "no epilogue is allowed.",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (L->getExitingBlock() != L->getLoopLatch() ||
+ LVL.hasUncountableEarlyExit()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported yet for early-exit loops",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ // We can apply tail-folding on the vectorized epilogue loop.
+ return CM_EpilogueNotNeededFoldTail;
+}
+
bool VFSelectionContext::isEpilogueVectorizationProfitable(
const ElementCount VF, const unsigned IC) const {
// FIXME: We need a much better cost-model to take different parameters such
@@ -5470,21 +5549,27 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// end.
buildVPlans(*VPlan1, UserVF, UserVF, *CM);
- ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
- if (EpilogueUserVF.isVector() &&
- ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
- CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
- buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
- }
- if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
- // For scalar VF, skip VPlan cost check as VPlan cost is designed for
- // vector VFs only.
- if (UserVF.isScalar() ||
- cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid()) {
- LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
- LLVM_DEBUG(printPlans(dbgs()));
- return;
+ // For scalar VF, skip VPlan cost check as VPlan cost is designed for
+ // vector VFs only.
+ if (!VPlans.empty() &&
+ (UserVF.isScalar() ||
+ cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid())) {
+ // Plan for epilogue only if we succeeded in building main loop vplan.
+
+ // Try to plan for tail-folded epilogue if it's enabled/doable,
+ // otherwise plan for unpredicated epilogue:
+ bool EpilogueTfPlanCreated = planForEpilogueTF();
+ if (!EpilogueTfPlanCreated) {
+ ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
+ if (EpilogueUserVF.isVector() &&
+ ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
+ CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
+ buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
+ }
}
+ LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
+ LLVM_DEBUG(printPlans(dbgs()));
+ return;
}
VPlans.clear();
reportVectorizationInfo("UserVF ignored because of invalid costs.",
@@ -5513,61 +5598,80 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
LLVM_DEBUG(printPlans(dbgs()));
}
-bool LoopVectorizationPlanner::planForEpilogueTF(
- LoopVectorizationCostModel &EpilogueCM) {
- if (VPlans.empty()) {
- LLVM_DEBUG(dbgs() << "LV: no vplans have been built for main loop VF, bail "
- "out of epilogue tail-folding\n");
+bool LoopVectorizationPlanner::planForEpilogueTF() {
+ if (!EpilogueTfCM)
return false;
- }
+ assert(EpilogueTfCM->preferTailFoldedLoop() &&
+ "Epilogue tail-folding is expected to be enabled");
- EpilogueCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
- EpilogueCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
+ LLVM_DEBUG(dbgs() << "LV: plan for tail-folded epilogue\n");
+
+ EpilogueTfCM->ValuesToIgnore.insert_range(CM->ValuesToIgnore);
+ EpilogueTfCM->VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EpilogueCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
- if (!MaxFactors ||
- !EpilogueCM
- .foldTailByMasking()) { // Cases that should not to be vectorized
- // or tail-folded.
+ EpilogueTfCM->computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
+ if (!MaxFactors || !EpilogueTfCM->foldTailByMasking()) {
+ // Cases that should not to be vectorized or tail-folded.
reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
return false;
}
- auto VPlan1 = tryToBuildVPlan1(EpilogueCM);
+ auto VPlan1 = tryToBuildVPlan1(*EpilogueTfCM);
+
+ // If we're here, the main loop's initial VPlan was built successfully.
+ // Building one for the tail-folded loop should therefore also succeed, since
+ // nothing tail-folding-specific happens yet at this point. Still check below
+ // to catch any unexpected failure.
+ if (!VPlan1) {
+ reportVectorizationInfo(
+ "Failed to build initial tail-folded epilogue VPlan",
+ "InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ return false;
+ }
if (!useMaskedInterleavedAccesses(TTI)) {
LLVM_DEBUG(
dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by "
"masking which requires masked-interleaved support.\n");
- if (EpilogueCM.InterleaveInfo.invalidateGroups())
+ if (EpilogueTfCM->InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- EpilogueCM.invalidateCostModelingDecisions();
+ EpilogueTfCM->invalidateCostModelingDecisions();
}
Legal->prepareToFoldTailByMasking();
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EpilogueCM.collectNonVectorizedAndSetWideningDecisions(
+ EpilogueTfCM->collectNonVectorizedAndSetWideningDecisions(
EpilogueVectorizationForceVF);
- // Expecting only 2 vplans as epilogue TF is applied only for forced VFs.
- assert(VPlans.size() == 2 &&
- "For tail-folded epilogue, VPlans size is expected to be 2");
- // Remove the last vplan, which should be the epilogue plan to replace it by
- // the tail-folded vplan:
- assert(VPlans.back()->getSingleVF() == EpilogueVectorizationForceVF &&
- "For tail-folded epilogue, last vplan is expected to have "
- "EpilogueUserVF");
- VPlans.pop_back();
+ size_t NumPlansBefore = VPlans.size();
buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
- EpilogueVectorizationForceVF, EpilogueCM);
+ EpilogueVectorizationForceVF, *EpilogueTfCM);
+
+ // Check that a vplan is successfully built:
+ if (VPlans.size() == NumPlansBefore ||
+ VPlans.back()->getSingleVF() != EpilogueVectorizationForceVF ||
+ !VPlans.back()->hasTailFolded()) {
+ reportVectorizationInfo(
+ "Failed to build a valid tail-folded epilogue VPlan",
+ "InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ return false;
+ }
- cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
- EpilogueCM);
+ if (!cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
+ *EpilogueTfCM)
+ .isValid()) {
+ VPlans.pop_back();
+ reportVectorizationInfo("This case of epilogue loop can't be tail-folded "
+ "- Invalid costs",
+ "InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ return false;
+ }
+ LLVM_DEBUG(dbgs() << "LV: Tail-folded epilogue VPlan is created\n");
return true;
}
@@ -5915,16 +6019,20 @@ LoopVectorizationPlanner::computeBestVF() {
LoopVectorizationPlanner::LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
- InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
+ std::unique_ptr<LoopVectorizationCostModel> CM,
+ std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM,
+ VFSelectionContext &Config, InterleavedAccessInfo &IAI,
+ PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE)
: OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+ CM(std::move(CM)), EpilogueTfCM(std::move(EpilogueTfCM)), Config(Config),
+ IAI(IAI), PSE(PSE), ORE(ORE) {}
LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
+void LoopVectorizationPlanner::clearEpilogueTfCM() { EpilogueTfCM.reset(); }
+
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT,
@@ -5952,8 +6060,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan, BestVF, VScale);
}
+ const bool IsTailFolded = BestVPlan.hasTailFolded();
if (vputils::findIncomingAliasMask(BestVPlan)) {
- assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
+ assert(IsTailFolded && "Expected tail folding to be enabled");
RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
*Legal->getRuntimePointerChecking()->getDiffChecks(),
HasBranchWeights);
@@ -5993,7 +6102,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
// Regions are dissolved after optimizing for VF and UF, which completely
// removes unneeded loop regions first.
- const bool HasTailFolded = BestVPlan.hasTailFolded();
RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
// Expand BranchOnTwoConds after dissolution, when latch has direct access to
// its successors.
@@ -6013,7 +6121,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
"loops not exiting via the latch without required epilogue?");
RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
- VectorPH, HasTailFolded, RequiresScalarEpilogue,
+ VectorPH, IsTailFolded, RequiresScalarEpilogue,
&BestVPlan.getVFxUF(), MaxRuntimeStep);
RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
BestVF);
@@ -7220,78 +7328,6 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
-/// Determine how to lower the epilogue for the vector epilogue loop.
-/// Check if there are any conflicts that prevent tail-folding the epilogue.
-/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
-/// otherwise CM_EpilogueAllowed.
-static EpilogueLowering
-getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
- OptimizationRemarkEmitter *ORE,
- LoopVectorizationLegality &LVL,
- LoopVectorizeHints &Hints) {
- // Epilogue TF is only enabled when explicitly requested via command line.
- if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
- EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
- return CM_EpilogueAllowed;
-
- if (!EnableEpilogueVectorization) {
- reportVectorizationInfo(
- "Options conflict, epilogue vectorization is disallowed while "
- "epilogue tail-folding allowed!",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
- reportVectorizationInfo("For now, epilogue tail-folding can't be "
- "applied without forced main/epilogue loop VF",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- if (ElementCount::isKnownLE(Hints.getWidth(), EpilogueVectorizationForceVF)) {
- reportVectorizationInfo("For now, epilogue tail-folding can't be applied "
- "when VF of the main loop <= VF of the epilogue",
- "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- if (!L->isInnermost()) {
- reportVectorizationInfo(
- "Epilogue tail-folding is not supported for outer loop",
- "InvalidTailFoldedEpilogue", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- // If scalar epilogue is explicitly required, we can't apply TF.
- if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- reportVectorizationInfo(
- "Epilogue tail-folding can't be applied because scalar epilogue is "
- "required. Fall back to a normal epilogue",
- "InvalidTailFoldedEpilogue", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- // If having epilogue is NOT allowed, then no epilogue to apply TF for.
- if (!MainCM.isEpilogueAllowed()) {
- reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
- "no epilogue is allowed.",
- "InvalidTailFoldedEpilogue", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- if (L->getExitingBlock() != L->getLoopLatch() ||
- LVL.hasUncountableEarlyExit()) {
- reportVectorizationInfo(
- "Epilogue tail-folding is not supported yet for early-exit loops",
- "InvalidTailFoldedEpilogue", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- // We can apply tail-folding on the vectorized epilogue loop.
- return CM_EpilogueNotNeededFoldTail;
-}
-
// Emit a remark if there are stores to floats that required a floating point
// extension. If the vectorized loop was generated with floating point there
// will be a performance penalty from the conversion overhead and the change in
@@ -7865,7 +7901,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove,
ArrayRef<VPInstruction *> ResumeValues,
- bool IsEpilogueTFEnabled) {
+ bool IsEpilogueTfEnabled) {
BasicBlock *VecEpilogueIterationCountCheck =
cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
BasicBlock *VecEpiloguePreHeader =
@@ -7895,7 +7931,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
// to, even a trip count too small for the epilogue VF is handled safely by
// the masked epilogue vector loop, so skip straight to its preheader.
RedirectEdge(EPI.EpilogueIterationCountCheck,
- IsEpilogueTFEnabled ? VecEpiloguePreHeader : ScalarPH);
+ IsEpilogueTfEnabled ? VecEpiloguePreHeader : ScalarPH);
// Adjust the terminators of runtime check blocks and phis using them.
BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
@@ -7932,12 +7968,12 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
}
// When the epilogue is tail-folded, EpilogueIterationCountCheck
// (iter.check) is redirected to branch straight into the vector epilogue
- // preheader (see the IsEpilogueTFEnabled redirect above), so it is now a
+ // preheader (see the IsEpilogueTfEnabled redirect above), so it is now a
// predecessor and its incoming value must be kept rather than stripped.
// TODO: revisit for reduction phis, whose resume value on this bypass
// edge may need dedicated handling rather than reusing the value already
// present here.
- if (!IsEpilogueTFEnabled)
+ if (!IsEpilogueTfEnabled)
Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
}
@@ -7957,7 +7993,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
if (Phi.use_empty())
Phi.eraseFromParent();
- if (IsEpilogueTFEnabled) {
+ if (IsEpilogueTfEnabled) {
// The epilogue vector loop is tail-folded, so it can safely handle
// any remaining iterations, including zero, via masking.
// vec.epilog.iter.check's own min-iters check was therefore built with a
@@ -8172,28 +8208,34 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- // Use the planner for vectorization.
- LoopVectorizationPlanner LVP(
- L, LI, DT, TLI, *TTI, &LVL,
- std::make_unique<LoopVectorizationCostModel>(
- SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
- Config, IAI, PSE, ORE);
-
+ auto CM = std::make_unique<LoopVectorizationCostModel>(
+ SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config);
+
+ // Setup the epilogue tail-folding CM. Only built when tail-folding the
+ // epilogue is actually a candidate, to avoid the cost of an extra
+ // InterleavedAccessInfo scan and LoopVectorizationCostModel construction
+ // for the common case where this (experimental, off-by-default) feature
+ // isn't in use.
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
- std::optional<InterleavedAccessInfo> TailFoldingCMIAI;
- std::optional<LoopVectorizationCostModel> EpilogueTailFoldingCM;
+ getEpilogueTailLowering(*CM, L, ORE, LVL, Hints);
+ std::optional<InterleavedAccessInfo> EpilogueTfCMIAI;
+ std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM;
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
- TailFoldingCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
+ EpilogueTfCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
if (UseInterleaved)
- TailFoldingCMIAI->analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
- EpilogueTailFoldingCM.emplace(CM_EpilogueNotNeededFoldTail, L, PSE, LI,
- &LVL, *TTI, TLI, AC, ORE, GetBFI, F,
- *TailFoldingCMIAI, Config);
+ EpilogueTfCMIAI->analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
+ EpilogueTfCM = std::make_unique<LoopVectorizationCostModel>(
+ EpilogueTailLoweringStatus, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
+ GetBFI, F, *EpilogueTfCMIAI, Config);
}
+ // Use the planner for vectorization.
+ LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, std::move(CM),
+ std::move(EpilogueTfCM), Config, IAI, PSE,
+ ORE);
+
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
@@ -8206,14 +8248,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- if (EpilogueTailFoldingCM)
- if (!LVP.planForEpilogueTF(EpilogueTailFoldingCM.value())) {
- // we can't apply epilogue TF:
- reportVectorizationInfo(
- "Applying epilogue tail-folding failed, disable it.",
- "InvalidTailFoldedEpilogue", ORE, L);
- EpilogueTailFoldingCM.reset();
- }
+ // Right now, after planning, the epilogue tail-folding CM is not needed
+ // anymore. Clear it.
+ LVP.clearEpilogueTfCM();
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
@@ -8488,12 +8525,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
BestMainPlan, BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
*PSE.getSE(), ResumeValues);
+ LVP.attachRuntimeChecks(BestEpiPlan, Checks, HasBranchWeights);
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
+ // Save the status of epilogue tail-folding:
+ const bool IsTailFolded = BestEpiPlan.hasTailFolded();
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LI, Checks, InstsToMove,
- ResumeValues, EpilogueTailFoldingCM.has_value());
+ ResumeValues, IsTailFolded);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 79495c4ad9c3e..ffde0d0084933 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,7 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; REQUIRES: asserts
; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug-only=loop-vectorize -mcpu=neoverse-v1 -S %s | FileCheck %s
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug-only=loop-vectorize -mattr=+sve -S %s | FileCheck %s
+
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width="vscale x 16" -epilogue-vectorization-force-VF="vscale x 8" -debug-only=loop-vectorize -mattr=+sve -S %s | FileCheck %s --check-prefix=CHECK-VS
; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize,vectorutils --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
@@ -31,7 +34,7 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -52,12 +55,68 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT5]], i64 [[N]])
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; CHECK-VS-LABEL: define void @test_epilogue_tf(
+; CHECK-VS-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 4
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[BROADCAST_SPLAT3]], ptr align 4 [[TMP10]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-VS-NEXT: [[INDEX_NEXT5]] = add i64 [[INDEX4]], [[TMP9]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT5]], i64 [[N]])
+; CHECK-VS-NEXT: [[TMP11:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; CHECK-VS-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: ret void
+;
entry:
br label %for.body
@@ -99,7 +158,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP5]], [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
@@ -125,7 +184,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
; CHECK-NEXT: br label %[[EXIT]]
@@ -133,6 +192,72 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
;
+; CHECK-VS-LABEL: define i32 @add_redc(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
+; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
+; CHECK-VS-NEXT: [[TMP8]] = add <vscale x 16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-VS-NEXT: [[TMP9]] = add <vscale x 16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[BIN_RDX:%.*]] = add <vscale x 16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[BIN_RDX]])
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[TMP14:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[TMP14]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP15]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[TMP16:%.*]] = add <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[TMP17]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP16]], <vscale x 8 x i32> [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
+; CHECK-VS-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP17]])
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[ADD_LCSSA]]
+;
entry:
br label %loop
@@ -176,7 +301,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP5]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD3]], <16 x i32> [[VEC_PHI2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[TMP4]], <16 x i32> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> [[RDX_MINMAX]])
@@ -203,7 +328,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
-; CHECK-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> [[TMP10]])
; CHECK-NEXT: br label %[[EXIT]]
@@ -211,6 +336,73 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[MAX_LCSSA]]
;
+; CHECK-VS-LABEL: define i32 @max_redc(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
+; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
+; CHECK-VS-NEXT: [[TMP8]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD]], <vscale x 16 x i32> [[VEC_PHI]])
+; CHECK-VS-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], <vscale x 16 x i32> [[VEC_PHI2]])
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.umax.nxv16i32(<vscale x 16 x i32> [[RDX_MINMAX]])
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i32> @llvm.umax.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_LOAD]], <vscale x 8 x i32> [[VEC_PHI5]])
+; CHECK-VS-NEXT: [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP15]], <vscale x 8 x i32> [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-VS-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32> [[TMP16]])
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[MAX_LCSSA]]
+;
entry:
br label %loop
@@ -248,7 +440,7 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[WIDE_LOAD]], i64 15
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -268,7 +460,7 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
-; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP8:%.*]] = xor <8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP8]], i1 false)
@@ -279,6 +471,66 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[LOAD_LCSSA]]
;
+; CHECK-VS-LABEL: define i32 @live-out(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP9:%.*]] = mul nuw i32 [[TMP8]], 16
+; CHECK-VS-NEXT: [[TMP10:%.*]] = sub i32 [[TMP9]], 1
+; CHECK-VS-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i32> [[WIDE_LOAD]], i32 [[TMP10]]
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX2]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], [[TMP13]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-VS-NEXT: [[TMP15:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP16:%.*]] = xor i1 [[TMP15]], true
+; CHECK-VS-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP17:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-VS-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP17]], i1 false)
+; CHECK-VS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-VS-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-VS-NEXT: br label %[[FOR_END]]
+; CHECK-VS: [[FOR_END]]:
+; CHECK-VS-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[LOAD_LCSSA]]
+;
entry:
br label %loop
@@ -333,7 +585,7 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-NEXT: store <16 x i32> [[REVERSE]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -358,7 +610,7 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT8]], i32 [[TMP2]])
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
-; CHECK-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
@@ -369,10 +621,102 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-NEXT: store i32 [[VAL]], ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[IV_NEXT]] = sub nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sge i32 [[IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; CHECK-VS-LABEL: define void @reversed-loop(
+; CHECK-VS-SAME: ptr [[A:%.*]], i32 [[N:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[ST:%.*]] = sub i32 [[N]], 1
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = add i32 [[N]], -2
+; CHECK-VS-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 -1)
+; CHECK-VS-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[SMIN1]]
+; CHECK-VS-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], [[TMP4]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK-VS: [[VECTOR_SCEVCHECK]]:
+; CHECK-VS-NEXT: [[TMP5:%.*]] = add i32 [[N]], -2
+; CHECK-VS-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP5]], i32 -1)
+; CHECK-VS-NEXT: [[TMP6:%.*]] = sub i32 [[TMP5]], [[SMIN]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = sub i32 [[ST]], [[TMP6]]
+; CHECK-VS-NEXT: [[TMP8:%.*]] = icmp sgt i32 [[TMP7]], [[ST]]
+; CHECK-VS-NEXT: br i1 [[TMP8]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP9:%.*]] = shl nuw i32 [[TMP3]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i32 [[TMP2]], [[TMP9]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP3]], 4
+; CHECK-VS-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP10]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], [[TMP11]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[TMP12:%.*]] = sub i32 [[ST]], [[N_VEC]]
+; CHECK-VS-NEXT: [[REVERSE:%.*]] = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> [[BROADCAST_SPLAT]])
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP13:%.*]] = sub i32 [[ST]], [[INDEX]]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]]
+; CHECK-VS-NEXT: [[TMP15:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-VS-NEXT: [[TMP16:%.*]] = sub nuw nsw i64 [[TMP15]], 1
+; CHECK-VS-NEXT: [[TMP17:%.*]] = sub i64 0, [[TMP16]]
+; CHECK-VS-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP17]]
+; CHECK-VS-NEXT: [[TMP19:%.*]] = sub i64 [[TMP17]], [[TMP15]]
+; CHECK-VS-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP19]]
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP18]], align 4
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP20]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP11]]
+; CHECK-VS-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP23:%.*]] = shl nuw i32 [[TMP22]], 3
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[REVERSE5:%.*]] = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> [[BROADCAST_SPLAT4]])
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 [[VEC_EPILOG_RESUME_VAL]], i32 [[TMP2]])
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP24:%.*]] = sub i32 [[ST]], [[INDEX6]]
+; CHECK-VS-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP24]]
+; CHECK-VS-NEXT: [[TMP26:%.*]] = zext i32 [[TMP23]] to i64
+; CHECK-VS-NEXT: [[TMP27:%.*]] = sub nuw nsw i64 [[TMP26]], 1
+; CHECK-VS-NEXT: [[TMP28:%.*]] = sub i64 0, [[TMP27]]
+; CHECK-VS-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[TMP25]], i64 [[TMP28]]
+; CHECK-VS-NEXT: [[REVERSE7:%.*]] = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[REVERSE5]], ptr align 4 [[TMP29]], <vscale x 8 x i1> [[REVERSE7]])
+; CHECK-VS-NEXT: [[INDEX_NEXT8]] = add i32 [[INDEX6]], [[TMP23]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 [[INDEX_NEXT8]], i32 [[TMP2]])
+; CHECK-VS-NEXT: [[TMP30:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP31:%.*]] = xor i1 [[TMP30]], true
+; CHECK-VS-NEXT: br i1 [[TMP31]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-VS: [[FOR_BODY]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i32 [ [[ST]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VS-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
+; CHECK-VS-NEXT: store i32 [[VAL]], ptr [[ARRAYIDX]], align 4
+; CHECK-VS-NEXT: [[IV_NEXT]] = sub nuw nsw i32 [[IV]], 1
+; CHECK-VS-NEXT: [[EXITCOND:%.*]] = icmp sge i32 [[IV_NEXT]], 0
+; CHECK-VS-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: ret void
+;
entry:
%st = sub i32 %n, 1
br label %for.body
@@ -389,72 +733,6 @@ exit:
ret void
}
-define void @math_func(ptr %A, i32 %n) {
-; CHECK-LABEL: define void @math_func(
-; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[UMAX]], 16
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = and i32 [[UMAX]], 15
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[UMAX]], [[TMP0]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = call <16 x float> @llvm.pow.v16f32(<16 x float> [[WIDE_LOAD]], <16 x float> splat (float 2.000000e+00))
-; CHECK-NEXT: store <16 x float> [[TMP2]], ptr [[TMP1]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
-; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[VEC_EPILOG_RESUME_VAL]], i32 [[UMAX]])
-; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i32 [[INDEX2]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x float> poison)
-; CHECK-NEXT: [[TMP5:%.*]] = call <8 x float> @llvm.pow.v8f32(<8 x float> [[WIDE_MASKED_LOAD]], <8 x float> splat (float 2.000000e+00))
-; CHECK-NEXT: call void @llvm.masked.store.v8f32.p0(<8 x float> [[TMP5]], ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT3]] = add i32 [[INDEX2]], 8
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT3]], i32 [[UMAX]])
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
-; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds float, ptr %A, i32 %iv
- %load = load float, ptr %arrayidx, align 4
- %val = call float @llvm.pow.f32(float %load, float 2.0)
- store float %val, ptr %arrayidx, align 4
- %iv.next = add nuw nsw i32 %iv, 1
- %exitcond = icmp ult i32 %iv.next, %n
- br i1 %exitcond, label %for.body, label %exit
-
-exit:
- ret void
-}
-
define i64 @test_no_masked_interleave_support(i64 %y, i32 %n) {
; CHECK-INVALIDATE-INTERLEAVE-LABEL: Checking a loop in 'test_no_masked_interleave_support'
; CHECK-INVALIDATE-INTERLEAVE: LV: epilogue tail-folding is enabled
@@ -481,3 +759,4 @@ cond.end:
for.cond.cleanup:
ret i64 %cond
}
+
>From de618172c620f5d422b792b179799e79e5af8edf Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 24 Aug 2026 20:21:28 +0100
Subject: [PATCH 12/16] Add supported reduction cases
---
.../AArch64/fold-epilogue-tail-reductions.ll | 584 ++++++++++++++++++
.../AArch64/fold-epilogue-tail.ll | 288 ---------
2 files changed, 584 insertions(+), 288 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
new file mode 100644
index 0000000000000..1b2088b617606
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
@@ -0,0 +1,584 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; REQUIRES: asserts
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mattr=+sve -S %s | FileCheck %s
+
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width="vscale x 16" -epilogue-vectorization-force-VF="vscale x 8" -mattr=+sve -S %s | FileCheck %s --check-prefix=CHECK-VS
+
+target triple = "aarch64-linux-gnu"
+
+define i32 @add_redc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @add_redc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4]] = add <16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[TMP8]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP9]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = add <8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
+; CHECK-NEXT: [[TMP11]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP10]], <8 x i32> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
+;
+; CHECK-VS-LABEL: define i32 @add_redc(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
+; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
+; CHECK-VS-NEXT: [[TMP8]] = add <vscale x 16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-VS-NEXT: [[TMP9]] = add <vscale x 16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[BIN_RDX:%.*]] = add <vscale x 16 x i32> [[TMP9]], [[TMP8]]
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[BIN_RDX]])
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[TMP14:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[TMP14]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP15]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[TMP16:%.*]] = add <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[TMP17]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP16]], <vscale x 8 x i32> [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
+; CHECK-VS-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP17]])
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[ADD_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %add = add i32 %load, %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %add
+}
+
+define i32 @max_redc(ptr %src, i64 %n) {
+; CHECK-LABEL: define i32 @max_redc(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD]], <16 x i32> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD3]], <16 x i32> [[VEC_PHI2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[TMP4]], <16 x i32> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> [[RDX_MINMAX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP8]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[WIDE_MASKED_LOAD]], <8 x i32> [[VEC_PHI5]])
+; CHECK-NEXT: [[TMP10]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP9]], <8 x i32> [[VEC_PHI5]]
+; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> [[TMP10]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[MAX_LCSSA]]
+;
+; CHECK-VS-LABEL: define i32 @max_redc(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
+; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
+; CHECK-VS-NEXT: [[TMP8]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD]], <vscale x 16 x i32> [[VEC_PHI]])
+; CHECK-VS-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], <vscale x 16 x i32> [[VEC_PHI2]])
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.umax.nxv16i32(<vscale x 16 x i32> [[RDX_MINMAX]])
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i32> @llvm.umax.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_LOAD]], <vscale x 8 x i32> [[VEC_PHI5]])
+; CHECK-VS-NEXT: [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP15]], <vscale x 8 x i32> [[VEC_PHI5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-VS-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32> [[TMP16]])
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[MAX_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %max, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %max = call i32 @llvm.umax(i32 %load, i32 %red)
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %max
+}
+
+define i64 @find_iv(ptr %src, i64 %n) {
+;
+; CHECK-LABEL: define i64 @find_iv(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 15
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4]] = or <16 x i1> [[VEC_PHI1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5]] = select <16 x i1> [[TMP3]], <16 x i64> [[VEC_IND]], <16 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP9]], i64 [[TMP7]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[ICMP:%.*]] = icmp eq i8 [[LOAD]], 0
+; CHECK-NEXT: [[SELECT]] = select i1 [[ICMP]], i64 [[IV]], i64 [[RED]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[SELECT_LCSSA:%.*]] = phi i64 [ [[SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SELECT_LCSSA]]
+;
+; CHECK-VS-LABEL: define i64 @find_iv(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ENTRY:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-VS-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-VS-NEXT: [[TMP7]] = or <vscale x 16 x i1> [[VEC_PHI1]], [[TMP6]]
+; CHECK-VS-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i64> [[VEC_IND]], <vscale x 16 x i64> [[VEC_PHI]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-VS-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-VS-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.umax.nxv16i64(<vscale x 16 x i64> [[TMP8]])
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP7]])
+; CHECK-VS-NEXT: [[TMP12:%.*]] = freeze i1 [[TMP11]]
+; CHECK-VS-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP12]], i64 [[TMP10]], i64 0
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VS: [[SCALAR_PH]]:
+; CHECK-VS-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-VS-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-VS-NEXT: [[ICMP:%.*]] = icmp eq i8 [[LOAD]], 0
+; CHECK-VS-NEXT: [[SELECT]] = select i1 [[ICMP]], i64 [[IV]], i64 [[RED]]
+; CHECK-VS-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-VS-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-VS-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[SELECT_LCSSA:%.*]] = phi i64 [ [[SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i64 [[SELECT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i64 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i8, ptr %src, i64 %iv
+ %load = load i8, ptr %gep, align 1
+ %icmp = icmp eq i8 %load, 0
+ %select = select i1 %icmp, i64 %iv, i64 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i64 %select
+}
+
+define i32 @any-of(ptr %src, i64 %n) {
+;
+; CHECK-LABEL: define i32 @any-of(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-NEXT: [[TMP6]] = or <16 x i1> [[VEC_PHI]], [[TMP4]]
+; CHECK-NEXT: [[TMP7]] = or <16 x i1> [[VEC_PHI2]], [[TMP5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = or <16 x i1> [[TMP7]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[BIN_RDX]])
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP9]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP10]], i32 1, i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP11]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i1> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP12]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <8 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP13]], <8 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP15]] = or <8 x i1> [[VEC_PHI5]], [[TMP14]]
+; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = xor i1 [[TMP16]], true
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP15]])
+; CHECK-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP18]]
+; CHECK-NEXT: [[RDX_SELECT7:%.*]] = select i1 [[TMP19]], i32 1, i32 0
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[SELECT_LCSSA]]
+;
+; CHECK-VS-LABEL: define i32 @any-of(
+; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 [[TMP4]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
+; CHECK-VS-NEXT: [[TMP8:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-VS-NEXT: [[TMP9:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-VS-NEXT: [[TMP10]] = or <vscale x 16 x i1> [[VEC_PHI]], [[TMP8]]
+; CHECK-VS-NEXT: [[TMP11]] = or <vscale x 16 x i1> [[VEC_PHI2]], [[TMP9]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[BIN_RDX:%.*]] = or <vscale x 16 x i1> [[TMP11]], [[TMP10]]
+; CHECK-VS-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[BIN_RDX]])
+; CHECK-VS-NEXT: [[TMP14:%.*]] = freeze i1 [[TMP13]]
+; CHECK-VS-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP14]], i32 1, i32 0
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP15:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
+; CHECK-VS-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 [[TMP15]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i1> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr align 1 [[TMP18]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
+; CHECK-VS-NEXT: [[TMP19:%.*]] = icmp eq <vscale x 8 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-VS-NEXT: [[TMP20:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i1> [[TMP19]], <vscale x 8 x i1> zeroinitializer
+; CHECK-VS-NEXT: [[TMP21]] = or <vscale x 8 x i1> [[VEC_PHI5]], [[TMP20]]
+; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP17]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
+; CHECK-VS-NEXT: [[TMP22:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP23:%.*]] = xor i1 [[TMP22]], true
+; CHECK-VS-NEXT: br i1 [[TMP23]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP21]])
+; CHECK-VS-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP24]]
+; CHECK-VS-NEXT: [[RDX_SELECT7:%.*]] = select i1 [[TMP25]], i32 1, i32 0
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[SELECT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i8, ptr %src, i64 %iv
+ %load = load i8, ptr %gep, align 1
+ %icmp = icmp eq i8 %load, 0
+ %select = select i1 %icmp, i32 1, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index ffde0d0084933..0518a67b21df8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -132,294 +132,6 @@ exit:
ret void
}
-define i32 @add_redc(ptr %src, i64 %n) {
-; CHECK-LABEL: define i32 @add_redc(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP4]] = add <16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP5]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
-; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
-; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[TMP8]], %[[VEC_EPILOG_PH]] ], [ [[TMP11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP9]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
-; CHECK-NEXT: [[TMP10:%.*]] = add <8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
-; CHECK-NEXT: [[TMP11]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP10]], <8 x i32> [[VEC_PHI5]]
-; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
-;
-; CHECK-VS-LABEL: define i32 @add_redc(
-; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
-; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
-; CHECK-VS: [[VECTOR_PH]]:
-; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
-; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-VS: [[VECTOR_BODY]]:
-; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
-; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
-; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
-; CHECK-VS-NEXT: [[TMP8]] = add <vscale x 16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-VS-NEXT: [[TMP9]] = add <vscale x 16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
-; CHECK-VS: [[MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[BIN_RDX:%.*]] = add <vscale x 16 x i32> [[TMP9]], [[TMP8]]
-; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[BIN_RDX]])
-; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
-; CHECK-VS: [[VEC_EPILOG_PH]]:
-; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
-; CHECK-VS-NEXT: [[TMP14:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[TMP14]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
-; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP15]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
-; CHECK-VS-NEXT: [[TMP16:%.*]] = add <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI5]]
-; CHECK-VS-NEXT: [[TMP17]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP16]], <vscale x 8 x i32> [[VEC_PHI5]]
-; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
-; CHECK-VS-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-VS-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
-; CHECK-VS-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
-; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP17]])
-; CHECK-VS-NEXT: br label %[[EXIT]]
-; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-VS-NEXT: ret i32 [[ADD_LCSSA]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %red = phi i32 [ 0, %entry ], [ %add, %loop ]
- %gep = getelementptr inbounds i32, ptr %src, i64 %iv
- %load = load i32, ptr %gep, align 1
- %add = add i32 %load, %red
- %iv.next = add i64 %iv, 1
- %icmp3 = icmp eq i64 %iv, %n
- br i1 %icmp3, label %exit, label %loop
-
-exit:
- ret i32 %add
-}
-
-define i32 @max_redc(ptr %src, i64 %n) {
-; CHECK-LABEL: define i32 @max_redc(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 31
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i32>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP4]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD]], <16 x i32> [[VEC_PHI]])
-; CHECK-NEXT: [[TMP5]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[WIDE_LOAD3]], <16 x i32> [[VEC_PHI2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x i32> @llvm.umax.v16i32(<16 x i32> [[TMP4]], <16 x i32> [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> [[RDX_MINMAX]])
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
-; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[TMP8]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
-; CHECK-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[WIDE_MASKED_LOAD]], <8 x i32> [[VEC_PHI5]])
-; CHECK-NEXT: [[TMP10]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP9]], <8 x i32> [[VEC_PHI5]]
-; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], 8
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
-; CHECK-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> [[TMP10]])
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[MAX_LCSSA]]
-;
-; CHECK-VS-LABEL: define i32 @max_redc(
-; CHECK-VS-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-VS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-VS-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 5
-; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
-; CHECK-VS: [[VECTOR_PH]]:
-; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
-; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-VS: [[VECTOR_BODY]]:
-; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-VS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP4]]
-; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 1
-; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
-; CHECK-VS-NEXT: [[TMP8]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD]], <vscale x 16 x i32> [[VEC_PHI]])
-; CHECK-VS-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], <vscale x 16 x i32> [[VEC_PHI2]])
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
-; CHECK-VS: [[MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[RDX_MINMAX:%.*]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-VS-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.umax.nxv16i32(<vscale x 16 x i32> [[RDX_MINMAX]])
-; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
-; CHECK-VS: [[VEC_EPILOG_PH]]:
-; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[TMP0]])
-; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
-; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[VEC_PHI5:%.*]] = phi <vscale x 8 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX4]]
-; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 1 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
-; CHECK-VS-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i32> @llvm.umax.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_LOAD]], <vscale x 8 x i32> [[VEC_PHI5]])
-; CHECK-VS-NEXT: [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> [[TMP15]], <vscale x 8 x i32> [[VEC_PHI5]]
-; CHECK-VS-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX4]], [[TMP13]]
-; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT6]], i64 [[TMP0]])
-; CHECK-VS-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-VS-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
-; CHECK-VS-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
-; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32> [[TMP16]])
-; CHECK-VS-NEXT: br label %[[EXIT]]
-; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-VS-NEXT: ret i32 [[MAX_LCSSA]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %red = phi i32 [ 0, %entry ], [ %max, %loop ]
- %gep = getelementptr inbounds i32, ptr %src, i64 %iv
- %load = load i32, ptr %gep, align 1
- %max = call i32 @llvm.umax(i32 %load, i32 %red)
- %iv.next = add i64 %iv, 1
- %icmp3 = icmp eq i64 %iv, %n
- br i1 %icmp3, label %exit, label %loop
-
-exit:
- ret i32 %max
-}
-
define i32 @live-out(ptr %src, i64 %n) {
; CHECK-LABEL: define i32 @live-out(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
>From 75fade40d47c0e2b1f9adb0959cc53a145d9ab7e Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 4 Sep 2026 18:14:59 +0100
Subject: [PATCH 13/16] rebase + fix crash of 1st recurrence + fix crash when
start value of WIDEN-CANONICAL-INDUCTION is not CANONICAL-IV
---
.../Transforms/Vectorize/LoopVectorize.cpp | 18 +-
.../Transforms/Vectorize/VPlanLowering.cpp | 34 +-
.../AArch64/fold-epilogue-tail-reductions.ll | 258 +++++++-
.../AArch64/fold-epilogue-tail.ll | 286 ++++++---
.../AArch64/partial-reduce-with-predicate.ll | 563 ++++++++++++++++++
...g-vectorization-fixed-order-recurrences.ll | 173 +++++-
.../LoopVectorize/fold-epilogue-tail.ll | 37 +-
7 files changed, 1236 insertions(+), 133 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 26b31b8c184d0..ce4aa984465b4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3133,7 +3133,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
for (const auto &Plan : VPlans) {
// Skip cost remarks when Plan is not compatible with the CM.
// Specifically for the case of epilogue tail-folded Plans.
- if (Plan->hasTailFolded() ^ CM->preferTailFoldedLoop())
+ if (Plan->hasTailFolded() != CM->preferTailFoldedLoop())
continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
@@ -5551,15 +5551,13 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// For scalar VF, skip VPlan cost check as VPlan cost is designed for
// vector VFs only.
- if (!VPlans.empty() &&
+ if (!VPlans.empty() && (VPlans.front()->getSingleVF() == UserVF) &&
(UserVF.isScalar() ||
cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid())) {
// Plan for epilogue only if we succeeded in building main loop vplan.
-
// Try to plan for tail-folded epilogue if it's enabled/doable,
// otherwise plan for unpredicated epilogue:
- bool EpilogueTfPlanCreated = planForEpilogueTF();
- if (!EpilogueTfPlanCreated) {
+ if (!planForEpilogueTF()) {
ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
if (EpilogueUserVF.isVector() &&
ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
@@ -5628,6 +5626,7 @@ bool LoopVectorizationPlanner::planForEpilogueTF() {
reportVectorizationInfo(
"Failed to build initial tail-folded epilogue VPlan",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ assert(false && "Failed to build initial tail-folded epilogue VPlan");
return false;
}
@@ -7674,7 +7673,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto *Increment = vputils::findCanonicalIVIncrement(Plan);
assert(Increment && "Must have a canonical IV increment at this point");
IV->replaceUsesWithIf(Add, [Add, Increment](VPUser &U, unsigned) {
- return &U != Add && &U != Increment && !isa<VPWidenCanonicalIVRecipe>(&U);
+ return &U != Add && &U != Increment;
});
VPInstruction *OffsetIVInc =
VPBuilder::getToInsertAfter(Increment).createAdd(Increment, VPV);
@@ -7797,11 +7796,16 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
VPInstruction::CanonicalIVIncrementForPart, {VPV, &Plan.getVF()}, {},
R.getDebugLoc(), "index.part.next");
auto *EntryALM = EntryBuilder.createNaryOp(
- VPInstruction::ActiveLaneMask,
+ VPInstruction::WideActiveLaneMask,
{EntryIncrement, Plan.getTripCount(), ALMMultiplier}, R.getDebugLoc(),
"active.lane.mask.entry");
cast<VPHeaderPHIRecipe>(&R)->setStartValue(EntryALM);
continue;
+ } else if (isa<VPFirstOrderRecurrencePHIRecipe>(&R)) {
+ auto *RecPhi = cast<VPFirstOrderRecurrencePHIRecipe>(&R);
+ VPInstruction *ResumeForEpi =
+ IRPhiToResumeForEpi.at(cast<PHINode>(RecPhi->getUnderlyingInstr()));
+ ResumeV = ResumeForEpi->getUnderlyingValue();
} else {
// Retrieve the induction resume value via ResumeForEpilogue.
PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index ddde0cd83a704..08cc0f57acce6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -45,12 +45,36 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
if (!LoopRegion)
return;
- auto *WideCanIV =
- findUserOf<VPWidenCanonicalIVRecipe>(LoopRegion->getCanonicalIV());
+ VPWidenCanonicalIVRecipe *WideCanIV = nullptr;
+ VPIRValue *StartValue = nullptr;
+ // VPWidenCanonicalIVRecipe is either a direct user of CanonicalIV or
+ // Add (CanonicalIV, resumeValue) (like the case for tail-folded epilogue).
+ for (auto *User : LoopRegion->getCanonicalIV()->users()) {
+ if (isa<VPWidenCanonicalIVRecipe>(User)) {
+ WideCanIV = cast<VPWidenCanonicalIVRecipe>(User);
+ StartValue = Plan.getZero(WideCanIV->getScalarType());
+ break;
+ }
+ if (isa<VPInstruction>(User)) {
+ auto *UserInstr = cast<VPInstruction>(User);
+ auto *It = find_if(UserInstr->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ if (It != UserInstr->user_end()) {
+ WideCanIV = cast<VPWidenCanonicalIVRecipe>(*It);
+ match(UserInstr, m_Add(m_VPValue(), m_VPIRValue(StartValue)));
+ assert(StartValue &&
+ "WIDEN-CANONICAL-INDUCTION is only expected to be reached "
+ "through the canonical IV directly, or through a single 'add "
+ "CanonicalIV, StartValue' introduced for epilogue-loop resume "
+ "values; found a different pattern here");
+ if (!StartValue)
+ return;
+ }
+ }
+ }
if (!WideCanIV)
return;
- Type *CanIVTy = LoopRegion->getCanonicalIVType();
+ Type *CanIVTy = WideCanIV->getScalarType();
// Replace the wide canonical IV with a scalar-iv-steps over the canonical
// IV.
@@ -58,7 +82,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
VPBuilder Builder(WideCanIV);
WideCanIV->replaceAllUsesWith(vputils::createScalarIVSteps(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
- nullptr, Plan.getZero(CanIVTy), Plan.getConstantInt(CanIVTy, 1),
+ nullptr, StartValue, Plan.getConstantInt(CanIVTy, 1),
WideCanIV->getDebugLoc(), Builder,
{static_cast<bool>(WideCanIV->getNoWrapFlags().HasNUW), false}));
WideCanIV->eraseFromParent();
@@ -108,7 +132,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
InductionDescriptor::getCanonicalIntInduction(CanIVTy, SE);
VPValue *StepV = Plan.getConstantInt(CanIVTy, 1);
auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
- /*IV=*/nullptr, Plan.getZero(CanIVTy), StepV, &Plan.getVF(), ID,
+ /*IV=*/nullptr, StartValue, StepV, &Plan.getVF(), ID,
WideCanIV->getNoWrapFlags(), WideCanIV->getDebugLoc());
NewWideIV->insertBefore(&*Header->getFirstNonPhi());
WideCanIV->replaceAllUsesWith(NewWideIV);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
index 1b2088b617606..1c9c05884b7d4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
@@ -82,8 +82,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
@@ -96,7 +95,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
; CHECK-VS-NEXT: [[TMP8]] = add <vscale x 16 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-VS-NEXT: [[TMP9]] = add <vscale x 16 x i32> [[WIDE_LOAD3]], [[VEC_PHI2]]
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
@@ -226,8 +225,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
@@ -240,7 +238,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i32>, ptr [[TMP7]], align 1
; CHECK-VS-NEXT: [[TMP8]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD]], <vscale x 16 x i32> [[VEC_PHI]])
; CHECK-VS-NEXT: [[TMP9]] = call <vscale x 16 x i32> @llvm.umax.nxv16i32(<vscale x 16 x i32> [[WIDE_LOAD3]], <vscale x 16 x i32> [[VEC_PHI2]])
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
@@ -356,11 +354,10 @@ define i64 @find_iv(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
-; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-VS-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
-; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP2]], i64 0
; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
@@ -373,7 +370,7 @@ define i64 @find_iv(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-VS-NEXT: [[TMP7]] = or <vscale x 16 x i1> [[VEC_PHI1]], [[TMP6]]
; CHECK-VS-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i64> [[VEC_IND]], <vscale x 16 x i64> [[VEC_PHI]]
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-VS-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-VS-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
@@ -504,8 +501,7 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-VS-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
@@ -520,7 +516,7 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[TMP9:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD3]], zeroinitializer
; CHECK-VS-NEXT: [[TMP10]] = or <vscale x 16 x i1> [[VEC_PHI]], [[TMP8]]
; CHECK-VS-NEXT: [[TMP11]] = or <vscale x 16 x i1> [[VEC_PHI2]], [[TMP9]]
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-VS-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
@@ -582,3 +578,239 @@ loop:
exit:
ret i32 %select
}
+
+define i64 @arg_min_first_index(ptr %arr, i64 %n, i64 %start) {
+; CHECK-LABEL: define i64 @arg_min_first_index(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]], i64 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 15
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i64> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <16 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = select <16 x i1> [[TMP2]], <16 x i64> [[VEC_IND]], <16 x i64> [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP4]] = call <16 x i64> @llvm.smin.v16i64(<16 x i64> [[WIDE_LOAD]], <16 x i64> [[VEC_PHI]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 16)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> [[TMP4]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i64> poison, i64 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT3]], <16 x i64> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <16 x i64> [[TMP4]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP7]], <16 x i64> [[TMP3]], <16 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP6]], [[START]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 0, i64 [[TMP9]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX5:%.*]] = phi i64 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = and i64 [[N]], 7
+; CHECK-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT7]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <8 x i64> poison, i64 [[BC_MERGE_RDX5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT9]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT11]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <8 x i64> [[BROADCAST_SPLAT12]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX13:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND14:%.*]] = phi <8 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI15:%.*]] = phi <8 x i64> [ [[BROADCAST_SPLAT8]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI16:%.*]] = phi <8 x i64> [ [[BROADCAST_SPLAT10]], %[[VEC_EPILOG_PH]] ], [ [[TMP15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX13]]
+; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i64>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp slt <8 x i64> [[WIDE_LOAD17]], [[VEC_PHI15]]
+; CHECK-NEXT: [[TMP15]] = select <8 x i1> [[TMP14]], <8 x i64> [[VEC_IND14]], <8 x i64> [[VEC_PHI16]]
+; CHECK-NEXT: [[TMP16]] = call <8 x i64> @llvm.smin.v8i64(<8 x i64> [[WIDE_LOAD17]], <8 x i64> [[VEC_PHI15]])
+; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX13]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT19]] = add nuw nsw <8 x i64> [[VEC_IND14]], splat (i64 8)
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT18]], [[N_VEC6]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> [[TMP16]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT20:%.*]] = insertelement <8 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT21:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT20]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq <8 x i64> [[TMP16]], [[BROADCAST_SPLAT21]]
+; CHECK-NEXT: [[TMP20:%.*]] = select <8 x i1> [[TMP19]], <8 x i64> [[TMP15]], <8 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> [[TMP20]])
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP18]], [[START]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i64 0, i64 [[TMP21]]
+; CHECK-NEXT: [[CMP_N22:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-NEXT: br i1 [[CMP_N22]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX23:%.*]] = phi i64 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX24:%.*]] = phi i64 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[MIN:%.*]] = phi i64 [ [[BC_MERGE_RDX23]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MIN_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[MIN_LOC:%.*]] = phi i64 [ [[BC_MERGE_RDX24]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MIN_LOC_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[IV]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP24]], [[MIN]]
+; CHECK-NEXT: [[MIN_LOC_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_LOC]]
+; CHECK-NEXT: [[MIN_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[TMP24]], i64 [[MIN]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[MIN_LOC_NEXT_LCSSA:%.*]] = phi i64 [ [[MIN_LOC_NEXT]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[MIN_LOC_NEXT_LCSSA]]
+;
+; CHECK-VS-LABEL: define i64 @arg_min_first_index(
+; CHECK-VS-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]], i64 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[START]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i64> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 16 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i64>, ptr [[TMP4]], align 8
+; CHECK-VS-NEXT: [[TMP5:%.*]] = icmp slt <vscale x 16 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-VS-NEXT: [[TMP6]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i64> [[VEC_IND]], <vscale x 16 x i64> [[VEC_PHI4]]
+; CHECK-VS-NEXT: [[TMP7]] = call <vscale x 16 x i64> @llvm.smin.nxv16i64(<vscale x 16 x i64> [[WIDE_LOAD]], <vscale x 16 x i64> [[VEC_PHI]])
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-VS-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT3]]
+; CHECK-VS-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smin.nxv16i64(<vscale x 16 x i64> [[TMP7]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 16 x i64> [[TMP7]], [[BROADCAST_SPLAT6]]
+; CHECK-VS-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i64> [[TMP6]], <vscale x 16 x i64> splat (i64 -1)
+; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umin.nxv16i64(<vscale x 16 x i64> [[TMP11]])
+; CHECK-VS-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP9]], [[START]]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 0, i64 [[TMP12]]
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
+; CHECK-VS-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i64 [ [[TMP14]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
+; CHECK-VS-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[N]], [[TMP16]]
+; CHECK-VS-NEXT: [[N_VEC9:%.*]] = sub i64 [[N]], [[N_MOD_VF8]]
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[BC_MERGE_RDX]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT10]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[BC_MERGE_RDX7]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT12]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT14]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[INDUCTION:%.*]] = add nuw nsw <vscale x 8 x i64> [[BROADCAST_SPLAT15]], [[TMP17]]
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT16]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT23:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_IND19:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT24:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI20:%.*]] = phi <vscale x 8 x i64> [ [[BROADCAST_SPLAT11]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VEC_PHI21:%.*]] = phi <vscale x 8 x i64> [ [[BROADCAST_SPLAT13]], %[[VEC_EPILOG_PH]] ], [ [[TMP20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX18]]
+; CHECK-VS-NEXT: [[WIDE_LOAD22:%.*]] = load <vscale x 8 x i64>, ptr [[TMP18]], align 8
+; CHECK-VS-NEXT: [[TMP19:%.*]] = icmp slt <vscale x 8 x i64> [[WIDE_LOAD22]], [[VEC_PHI20]]
+; CHECK-VS-NEXT: [[TMP20]] = select <vscale x 8 x i1> [[TMP19]], <vscale x 8 x i64> [[VEC_IND19]], <vscale x 8 x i64> [[VEC_PHI21]]
+; CHECK-VS-NEXT: [[TMP21]] = call <vscale x 8 x i64> @llvm.smin.nxv8i64(<vscale x 8 x i64> [[WIDE_LOAD22]], <vscale x 8 x i64> [[VEC_PHI20]])
+; CHECK-VS-NEXT: [[INDEX_NEXT23]] = add nuw i64 [[INDEX18]], [[TMP16]]
+; CHECK-VS-NEXT: [[VEC_IND_NEXT24]] = add nuw nsw <vscale x 8 x i64> [[VEC_IND19]], [[BROADCAST_SPLAT17]]
+; CHECK-VS-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT23]], [[N_VEC9]]
+; CHECK-VS-NEXT: br i1 [[TMP22]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.smin.nxv8i64(<vscale x 8 x i64> [[TMP21]])
+; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT25:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP23]], i64 0
+; CHECK-VS-NEXT: [[BROADCAST_SPLAT26:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT25]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-VS-NEXT: [[TMP24:%.*]] = icmp eq <vscale x 8 x i64> [[TMP21]], [[BROADCAST_SPLAT26]]
+; CHECK-VS-NEXT: [[TMP25:%.*]] = select <vscale x 8 x i1> [[TMP24]], <vscale x 8 x i64> [[TMP20]], <vscale x 8 x i64> splat (i64 -1)
+; CHECK-VS-NEXT: [[TMP26:%.*]] = call i64 @llvm.vector.reduce.umin.nxv8i64(<vscale x 8 x i64> [[TMP25]])
+; CHECK-VS-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP23]], [[START]]
+; CHECK-VS-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i64 0, i64 [[TMP26]]
+; CHECK-VS-NEXT: [[CMP_N27:%.*]] = icmp eq i64 [[N]], [[N_VEC9]]
+; CHECK-VS-NEXT: br i1 [[CMP_N27]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX28:%.*]] = phi i64 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[BC_MERGE_RDX29:%.*]] = phi i64 [ [[TMP28]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-VS-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-VS: [[FOR_BODY]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VS-NEXT: [[MIN:%.*]] = phi i64 [ [[BC_MERGE_RDX28]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MIN_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VS-NEXT: [[MIN_LOC:%.*]] = phi i64 [ [[BC_MERGE_RDX29]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MIN_LOC_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VS-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[IV]]
+; CHECK-VS-NEXT: [[TMP29:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-VS-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP29]], [[MIN]]
+; CHECK-VS-NEXT: [[MIN_LOC_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_LOC]]
+; CHECK-VS-NEXT: [[MIN_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[TMP29]], i64 [[MIN]])
+; CHECK-VS-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VS-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]]
+; CHECK-VS: [[FOR_END]]:
+; CHECK-VS-NEXT: [[MIN_LOC_NEXT_LCSSA:%.*]] = phi i64 [ [[MIN_LOC_NEXT]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ [[TMP28]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i64 [[MIN_LOC_NEXT_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %min = phi i64 [ %start, %entry ], [ %min.next, %for.body ]
+ %min.loc = phi i64 [ 0, %entry ], [ %min.loc.next, %for.body ]
+ %arrayidx = getelementptr inbounds nuw [8 x i8], ptr %arr, i64 %iv
+ %0 = load i64, ptr %arrayidx
+ %cmp = icmp slt i64 %0, %min
+ %min.loc.next = select i1 %cmp, i64 %iv, i64 %min.loc
+ %min.next = tail call i64 @llvm.smin.i64(i64 %0, i64 %min)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret i64 %min.loc.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 0518a67b21df8..3cee861a822b4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,12 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; REQUIRES: asserts
-; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -debug-only=loop-vectorize -mattr=+sve -S %s | FileCheck %s
-; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width="vscale x 16" -epilogue-vectorization-force-VF="vscale x 8" -debug-only=loop-vectorize -mattr=+sve -S %s | FileCheck %s --check-prefix=CHECK-VS
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mattr=+sve %s | FileCheck %s
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize,vectorutils --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width="vscale x 16" -epilogue-vectorization-force-VF="vscale x 8" -mattr=+sve %s | FileCheck %s --check-prefix=CHECK-VS
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -debug-only=loop-vectorize,vectorutils --disable-output \
; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
target triple = "aarch64-linux-gnu"
@@ -74,21 +73,20 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[VAL]], i64 0
; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-VS-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP3]]
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 4
; CHECK-VS-NEXT: store <vscale x 16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 4
-; CHECK-VS-NEXT: store <vscale x 16 x i32> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-VS-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -96,8 +94,8 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
+; CHECK-VS-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[VAL]], i64 0
; CHECK-VS-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
@@ -105,13 +103,13 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX4]]
-; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[BROADCAST_SPLAT3]], ptr align 4 [[TMP10]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-VS-NEXT: [[INDEX_NEXT5]] = add i64 [[INDEX4]], [[TMP9]]
+; CHECK-VS-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX4]]
+; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[BROADCAST_SPLAT3]], ptr align 4 [[TMP9]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-VS-NEXT: [[INDEX_NEXT5]] = add i64 [[INDEX4]], [[TMP8]]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT5]], i64 [[N]])
-; CHECK-VS-NEXT: [[TMP11:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-VS-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
-; CHECK-VS-NEXT: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-VS-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: br label %[[EXIT]]
; CHECK-VS: [[EXIT]]:
@@ -196,51 +194,50 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-VS-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-VS-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 [[TMP3]]
-; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 4
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-VS-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP4]], i64 [[TMP3]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP5]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VS-NEXT: [[TMP9:%.*]] = mul nuw i32 [[TMP8]], 16
-; CHECK-VS-NEXT: [[TMP10:%.*]] = sub i32 [[TMP9]], 1
-; CHECK-VS-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i32> [[WIDE_LOAD]], i32 [[TMP10]]
+; CHECK-VS-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 16
+; CHECK-VS-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 1
+; CHECK-VS-NEXT: [[TMP10:%.*]] = extractelement <vscale x 16 x i32> [[WIDE_LOAD]], i32 [[TMP9]]
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
+; CHECK-VS-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX2]]
-; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
-; CHECK-VS-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], [[TMP13]]
+; CHECK-VS-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX2]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP13]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], [[TMP12]]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
-; CHECK-VS-NEXT: [[TMP15:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-VS-NEXT: [[TMP16:%.*]] = xor i1 [[TMP15]], true
-; CHECK-VS-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
+; CHECK-VS-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-VS-NEXT: [[TMP17:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; CHECK-VS-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP17]], i1 false)
+; CHECK-VS-NEXT: [[TMP16:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-VS-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP16]], i1 false)
; CHECK-VS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-VS-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-VS-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-VS-NEXT: br label %[[FOR_END]]
; CHECK-VS: [[FOR_END]]:
-; CHECK-VS-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[LOAD_LCSSA]]
;
entry:
@@ -258,6 +255,148 @@ for.end:
ret i32 %load
}
+define i32 @live_out_recurrence(ptr %A, i64 %n) {
+; CHECK-LABEL: define i32 @live_out_recurrence(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 16
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <16 x i32> [[WIDE_LOAD]], i64 14
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[WIDE_LOAD]], i64 15
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <8 x i32> poison, i32 [[SCALAR_RECUR_INIT]], i32 7
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> poison)
+; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[VECTOR_RECUR]], <8 x i32> [[WIDE_MASKED_LOAD]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; CHECK-NEXT: [[TMP8:%.*]] = xor <8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP8]], i1 false)
+; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP7]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[FOR_LCSSA]]
+;
+; CHECK-VS-LABEL: define i32 @live_out_recurrence(
+; CHECK-VS-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-VS-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-VS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-VS: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-VS-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 5
+; CHECK-VS-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-VS: [[VECTOR_PH]]:
+; CHECK-VS-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VS: [[VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VS-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP3]]
+; CHECK-VS-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[TMP5]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-VS-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS: [[MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 16
+; CHECK-VS-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 2
+; CHECK-VS-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 16 x i32> [[WIDE_LOAD]], i32 [[TMP9]]
+; CHECK-VS-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP11:%.*]] = mul nuw i32 [[TMP10]], 16
+; CHECK-VS-NEXT: [[TMP12:%.*]] = sub i32 [[TMP11]], 1
+; CHECK-VS-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 16 x i32> [[WIDE_LOAD]], i32 [[TMP12]]
+; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS: [[VEC_EPILOG_PH]]:
+; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-VS-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VS-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 3
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-VS-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP16:%.*]] = mul nuw i32 [[TMP15]], 8
+; CHECK-VS-NEXT: [[TMP17:%.*]] = sub i32 [[TMP16]], 1
+; CHECK-VS-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[SCALAR_RECUR_INIT]], i32 [[TMP17]]
+; CHECK-VS-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-VS-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VS-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX2]]
+; CHECK-VS-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP18]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
+; CHECK-VS-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX2]], [[TMP14]]
+; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT3]], i64 [[N]])
+; CHECK-VS-NEXT: [[TMP19:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
+; CHECK-VS-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-VS-NEXT: [[TMP21:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.right.nxv8i32(<vscale x 8 x i32> [[VECTOR_RECUR]], <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-VS-NEXT: [[TMP22:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-VS-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP22]], i1 false)
+; CHECK-VS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-VS-NEXT: [[TMP23:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[EXIT]]:
+; CHECK-VS-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: ret i32 [[FOR_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %for = phi i32 [ 0, %entry ], [ %l, %loop ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %for
+}
+
define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-LABEL: define void @reversed-loop(
; CHECK-SAME: ptr [[A:%.*]], i32 [[N:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {
@@ -362,29 +501,28 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-VS-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
; CHECK-VS: [[VECTOR_PH]]:
; CHECK-VS-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP3]], 4
-; CHECK-VS-NEXT: [[TMP11:%.*]] = shl nuw i32 [[TMP10]], 1
-; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], [[TMP11]]
+; CHECK-VS-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], [[TMP9]]
; CHECK-VS-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[VAL]], i64 0
; CHECK-VS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-VS-NEXT: [[TMP12:%.*]] = sub i32 [[ST]], [[N_VEC]]
+; CHECK-VS-NEXT: [[TMP11:%.*]] = sub i32 [[ST]], [[N_VEC]]
; CHECK-VS-NEXT: [[REVERSE:%.*]] = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> [[BROADCAST_SPLAT]])
; CHECK-VS-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VS: [[VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP13:%.*]] = sub i32 [[ST]], [[INDEX]]
-; CHECK-VS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]]
-; CHECK-VS-NEXT: [[TMP15:%.*]] = zext i32 [[TMP10]] to i64
-; CHECK-VS-NEXT: [[TMP16:%.*]] = sub nuw nsw i64 [[TMP15]], 1
-; CHECK-VS-NEXT: [[TMP17:%.*]] = sub i64 0, [[TMP16]]
-; CHECK-VS-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP17]]
-; CHECK-VS-NEXT: [[TMP19:%.*]] = sub i64 [[TMP17]], [[TMP15]]
-; CHECK-VS-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP19]]
-; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP18]], align 4
-; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP20]], align 4
-; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP11]]
-; CHECK-VS-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VS-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-VS-NEXT: [[TMP12:%.*]] = sub i32 [[ST]], [[INDEX]]
+; CHECK-VS-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP12]]
+; CHECK-VS-NEXT: [[TMP14:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-VS-NEXT: [[TMP15:%.*]] = sub nuw nsw i64 [[TMP14]], 1
+; CHECK-VS-NEXT: [[TMP16:%.*]] = sub i64 0, [[TMP15]]
+; CHECK-VS-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP16]]
+; CHECK-VS-NEXT: [[TMP18:%.*]] = sub i64 [[TMP16]], [[TMP14]]
+; CHECK-VS-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP18]]
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP17]], align 4
+; CHECK-VS-NEXT: store <vscale x 16 x i32> [[REVERSE]], ptr [[TMP19]], align 4
+; CHECK-VS-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]]
+; CHECK-VS-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VS-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-VS: [[MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -392,8 +530,8 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VS-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VS-NEXT: [[TMP23:%.*]] = shl nuw i32 [[TMP22]], 3
+; CHECK-VS-NEXT: [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-VS-NEXT: [[TMP22:%.*]] = shl nuw i32 [[TMP21]], 3
; CHECK-VS-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[VAL]], i64 0
; CHECK-VS-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
; CHECK-VS-NEXT: [[REVERSE5:%.*]] = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> [[BROADCAST_SPLAT4]])
@@ -402,19 +540,19 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-VS: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-VS-NEXT: [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VS-NEXT: [[TMP24:%.*]] = sub i32 [[ST]], [[INDEX6]]
-; CHECK-VS-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP24]]
-; CHECK-VS-NEXT: [[TMP26:%.*]] = zext i32 [[TMP23]] to i64
-; CHECK-VS-NEXT: [[TMP27:%.*]] = sub nuw nsw i64 [[TMP26]], 1
-; CHECK-VS-NEXT: [[TMP28:%.*]] = sub i64 0, [[TMP27]]
-; CHECK-VS-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[TMP25]], i64 [[TMP28]]
+; CHECK-VS-NEXT: [[TMP23:%.*]] = sub i32 [[ST]], [[INDEX6]]
+; CHECK-VS-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP23]]
+; CHECK-VS-NEXT: [[TMP25:%.*]] = zext i32 [[TMP22]] to i64
+; CHECK-VS-NEXT: [[TMP26:%.*]] = sub nuw nsw i64 [[TMP25]], 1
+; CHECK-VS-NEXT: [[TMP27:%.*]] = sub i64 0, [[TMP26]]
+; CHECK-VS-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP24]], i64 [[TMP27]]
; CHECK-VS-NEXT: [[REVERSE7:%.*]] = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[REVERSE5]], ptr align 4 [[TMP29]], <vscale x 8 x i1> [[REVERSE7]])
-; CHECK-VS-NEXT: [[INDEX_NEXT8]] = add i32 [[INDEX6]], [[TMP23]]
+; CHECK-VS-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[REVERSE5]], ptr align 4 [[TMP28]], <vscale x 8 x i1> [[REVERSE7]])
+; CHECK-VS-NEXT: [[INDEX_NEXT8]] = add i32 [[INDEX6]], [[TMP22]]
; CHECK-VS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 [[INDEX_NEXT8]], i32 [[TMP2]])
-; CHECK-VS-NEXT: [[TMP30:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-VS-NEXT: [[TMP31:%.*]] = xor i1 [[TMP30]], true
-; CHECK-VS-NEXT: br i1 [[TMP31]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-VS-NEXT: [[TMP29:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-VS-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
+; CHECK-VS-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: br label %[[EXIT]]
; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 0a1eddf6e2804..de28bb59b6bab 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -tail-folding-policy=must-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-TAILFOLD
+; RUN: opt -passes=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 -epilogue-tail-folding-policy=prefer-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-TAILFOLD-EPILOGUE
target triple = "aarch64-none-unknown-elf"
@@ -80,6 +81,79 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP10]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @pred_reduction(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i32> [ [[TMP13]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP14]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD8]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP15]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP17]], <8 x i1> [[TMP16]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD9]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP18]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE10]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI7]], <8 x i32> [[TMP19]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT11]] = add i64 [[INDEX6]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
entry:
br label %for.body
@@ -184,6 +258,79 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP10]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @pred_reduction_sext(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = sext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i32> [ [[TMP13]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP14]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD8]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP15]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP17]], <8 x i1> [[TMP16]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD9]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP18]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE10]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI7]], <8 x i32> [[TMP19]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT11]] = add i64 [[INDEX6]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
entry:
br label %for.body
@@ -300,6 +447,91 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP13]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP7]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP8]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD5]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = mul nuw nsw <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP12]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD6]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = mul nuw nsw <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP16]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI9:%.*]] = phi <2 x i32> [ [[TMP19]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE13:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP20]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD10]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP21]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP23]], <8 x i1> [[TMP22]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP24]], <8 x i1> [[TMP22]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP25:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD11]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP26:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD12]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = select <8 x i1> [[TMP22]], <8 x i32> [[TMP27]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE13]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI9]], <8 x i32> [[TMP28]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT14]] = add i64 [[INDEX8]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT14]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE13]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
entry:
br label %for.body
@@ -422,6 +654,92 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @pred_sub_reduction(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP7]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP8]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD5]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = mul nuw nsw <16 x i32> [[TMP9]], [[TMP10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP12]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD6]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = mul nuw nsw <16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP15]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP16]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = sub i32 0, [[TMP18]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI9:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE13:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP20]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD10]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP21]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP23]], <8 x i1> [[TMP22]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP24]], <8 x i1> [[TMP22]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP25:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD11]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP26:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD12]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = select <8 x i1> [[TMP22]], <8 x i32> [[TMP27]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE13]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI9]], <8 x i32> [[TMP28]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT14]] = add i64 [[INDEX8]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT14]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE13]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP32:%.*]] = sub i32 [[BC_MERGE_RDX]], [[TMP31]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP32]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
entry:
br label %for.body
@@ -543,6 +861,92 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP13]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @chained_pred_reduction(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = zext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE8]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP13]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = zext <16 x i8> [[WIDE_LOAD7]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE9]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE5]], <16 x i32> [[TMP14]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE9]], [[PARTIAL_REDUCE8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI11:%.*]] = phi <2 x i32> [ [[TMP17]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP18]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD12]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP19]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD13:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP21]], <8 x i1> [[TMP20]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD13]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP23:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP22]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE14:%.*]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI11]], <8 x i32> [[TMP23]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP24]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP25:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD15]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP26:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP25]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE16]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[PARTIAL_REDUCE14]], <8 x i32> [[TMP26]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT17]] = add i64 [[INDEX10]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT17]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP27]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
entry:
br label %for.body
@@ -664,6 +1068,92 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP12]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @reduction_before_pred(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP3]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD3]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE4:%.*]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP4]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD5]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD6]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP9]], <16 x i1> [[TMP7]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP10]], <16 x i1> [[TMP8]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = select <16 x i1> [[TMP7]], <16 x i32> [[TMP11]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE8]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP12]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD7]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP13]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE9]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE4]], <16 x i32> [[TMP14]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE9]], [[PARTIAL_REDUCE8]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI11:%.*]] = phi <2 x i32> [ [[TMP17]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP18]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD12]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE13:%.*]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI11]], <8 x i32> [[TMP20]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP21]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD14]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP23:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP22]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX10]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP24]], <8 x i1> [[TMP23]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP25:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD15]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP26:%.*]] = select <8 x i1> [[TMP23]], <8 x i32> [[TMP25]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE16]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[PARTIAL_REDUCE13]], <8 x i32> [[TMP26]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT17]] = add i64 [[INDEX10]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT17]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP27]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
entry:
br label %for.body
@@ -780,6 +1270,79 @@ define i32 @pred_reduction_incoming_1(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
;
+; CHECK-TAILFOLD-EPILOGUE-LABEL: define i32 @pred_reduction_incoming_1(
+; CHECK-TAILFOLD-EPILOGUE-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 31
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ne <16 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[TMP3]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP6]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP8:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP7]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP9:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD4]] to <16 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP10:%.*]] = select <16 x i1> [[TMP4]], <16 x i32> [[TMP9]], <16 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VEC_EPILOG_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_PHI7:%.*]] = phi <2 x i32> [ [[TMP13]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP14]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp ne <8 x i8> [[WIDE_MASKED_LOAD8]], zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP15]], <8 x i1> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX6]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP17]], <8 x i1> [[TMP16]], <8 x i8> poison)
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD9]] to <8 x i32>
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP19:%.*]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP18]], <8 x i32> zeroinitializer
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE10]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI7]], <8 x i32> [[TMP19]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT11]] = add i64 [[INDEX6]], 8
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
index 1ac73e9d6bf74..b2d5b6a3322ea 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=8 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=8 -epilogue-vectorization-force-VF=4 -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-target-supports-masked-memory-ops -S %s | FileCheck %s --check-prefix=CHECK-TAILFOLDED-EPILOGUE
define void @dead_for(ptr %a, i64 %N) {
@@ -66,6 +68,62 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; CHECK-TAILFOLDED-EPILOGUE-LABEL: define void @dead_for(
+; CHECK-TAILFOLDED-EPILOGUE-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP2:%.*]] = add <8 x i64> [[WIDE_LOAD]], splat (i64 10)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store <8 x i64> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-TAILFOLDED-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP4:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP5:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_VEC2:%.*]] = sub i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDUCTION:%.*]] = add nuw <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX3]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP6]], <4 x i64> poison)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP8:%.*]] = add <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 10)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP7]], <4 x i1> [[TMP6]])
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX_NEXT4]] = add i64 [[INDEX3]], 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT4]], [[N_VEC2]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLDED-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: ret void
+;
entry:
br label %loop
@@ -129,6 +187,50 @@ define i64 @for_phi_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
; CHECK-NEXT: ret i64 [[RES]]
;
+; CHECK-TAILFOLDED-EPILOGUE-LABEL: define i64 @for_phi_used_in_loop_and_live_out(
+; CHECK-TAILFOLDED-EPILOGUE-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ENTRY:.*]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR:%.*]] = phi <8 x i64> [ <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 99>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[WIDE_LOAD]] = load <8 x i64>, ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[VECTOR_RECUR]], <8 x i64> [[WIDE_LOAD]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store <8 x i64> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-TAILFOLDED-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 6
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-TAILFOLDED-EPILOGUE: [[SCALAR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 99, %[[ENTRY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[LOOP:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[LOOP]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store i64 [[FOR]], ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-TAILFOLDED-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: ret i64 [[RES]]
+;
entry:
br label %loop
@@ -191,7 +293,7 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[GEP]], align 4
; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 2
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 3
@@ -217,6 +319,75 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
; CHECK-NEXT: ret i64 [[RES]]
;
+; CHECK-TAILFOLDED-EPILOGUE-LABEL: define i64 @for_phi_not_used_in_loop_and_live_out(
+; CHECK-TAILFOLDED-EPILOGUE-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP0:%.*]] = and i64 [[N]], 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VECTOR_BODY]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP2:%.*]] = add <8 x i64> [[WIDE_LOAD]], splat (i64 10)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store <8 x i64> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-TAILFOLDED-EPILOGUE: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 6
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP4:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_VEC2:%.*]] = sub i64 [[N_RND_UP]], [[TMP4]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDUCTION:%.*]] = add nuw <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i64> poison, i64 [[SCALAR_RECUR_INIT]], i32 3
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ [[VECTOR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP5:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX3]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[WIDE_MASKED_LOAD]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 4 [[TMP6]], <4 x i1> [[TMP5]], <4 x i64> poison)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP7:%.*]] = add <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 10)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP6]], <4 x i1> [[TMP5]])
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[INDEX_NEXT4]] = add i64 [[INDEX3]], 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT4]], [[N_VEC2]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP8]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[WIDE_MASKED_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP10]], i1 false)
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLDED-EPILOGUE: [[EXIT]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[TMP11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[TMP12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: ret i64 [[RES]]
+;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index e96850b65316c..7de7b3e94f4f1 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -14,17 +14,14 @@
; RUN: %{cmd} -force-vector-width=8 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-VFs
+; RUN: %{cmd} -force-vector-width=8 -epilogue-vectorization-force-VF=16 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-BIGER-EPILOGUE
+
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-early-exit-vectorization-with-side-effects \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EARLY-EXIT
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 -vectorize-scev-check-threshold=0 < %s 2>&1 | FileCheck %s \
-; RUN: --check-prefix=CHECK-NO-VPLANS
-
-
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is enabled
@@ -41,6 +38,8 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
;
+; CHECK-INVALID-BIGER-EPILOGUE-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-INVALID-BIGER-EPILOGUE: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
entry:
br label %for.body
@@ -63,7 +62,6 @@ define void @test_no_iterations_left(ptr %A) {
; CHECK-LABEL: LV: Checking a loop in 'test_no_iterations_left'
; CHECK: LV: epilogue tail-folding is enabled
; CHECK: LV: This case of epilogue loop can't be tail-folded.
-; CHECK: LV: Applying epilogue tail-folding failed, disable it.
;
entry:
br label %for.body
@@ -198,33 +196,6 @@ exit:
ret void
}
-; Can't build a valid vplan for this case because too many SCEV checks needed,
-; more than the specfied limit.
-define i64 @test_no_vplan_built(ptr %dst, i64 %n) {
-; CHECK-NO-VPLANS-LABEL: Checking a loop in 'test_no_vplan_built'
-; CHECK-NO-VPLANS: LV: epilogue tail-folding is enabled
-; CHECK-NO-VPLANS: LV: no vplans have been built for main loop VF, bail out of epilogue tail-folding
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %dead.iv = phi i16 [ 0, %entry ], [ %dead.iv.next, %loop ]
- %prev = phi i64 [ 0, %entry ], [ %ext, %loop ]
- %iv.next = add nuw nsw i64 %iv, 1
- %dead.iv.next = add i16 %dead.iv, 1
- %ext = zext i16 %dead.iv.next to i64
- %gep = getelementptr inbounds i64, ptr %dst, i64 %prev
- store i64 %iv, ptr %gep, align 8
- %cmp = icmp slt i64 %iv.next, %n
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %result = phi i64 [ %ext, %loop ]
- ret i64 %result
-}
-
define void @test_outer_loop(ptr %A, i64 %m) {
; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
>From 4275d169d8eb436130245de7580bc7e904d5ac78 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Sat, 5 Sep 2026 04:25:11 +0100
Subject: [PATCH 14/16] format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ce4aa984465b4..47a8cd9a24b13 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5866,9 +5866,10 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
return Cost;
}
-InstructionCost LoopVectorizationPlanner::cost(
- VPlan &Plan, ElementCount VF, VPRegisterUsage *RU,
- LoopVectorizationCostModel &EnabledCM) const {
+InstructionCost
+LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
+ VPRegisterUsage *RU,
+ LoopVectorizationCostModel &EnabledCM) const {
VPCostContext CostCtx(*TLI, Plan, EnabledCM, Config,
/*ReusePrintingSlotTracker=*/true);
InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
@@ -8237,8 +8238,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the planner for vectorization.
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, std::move(CM),
- std::move(EpilogueTfCM), Config, IAI, PSE,
- ORE);
+ std::move(EpilogueTfCM), Config, IAI, PSE, ORE);
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
@@ -8468,8 +8468,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestPlan = *BestPlanPtr;
// Consider vectorizing the epilogue too if it's profitable.
- std::unique_ptr<VPlan> EpiPlan = LVP.selectBestEpiloguePlan(
- BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
+ std::unique_ptr<VPlan> EpiPlan =
+ LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
bool HasBranchWeights =
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
if (EpiPlan) {
>From 953c0cd969a9e0bdbad2a6fa0c01210c78014bb0 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Sat, 5 Sep 2026 06:52:50 +0100
Subject: [PATCH 15/16] exclude partial-alias
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++++++-
llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll | 6 ++++++
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 47a8cd9a24b13..2e8ad09f89a72 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3427,6 +3427,13 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (ForcePartialAliasingVectorization) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with alias masking",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
// We can apply tail-folding on the vectorized epilogue loop.
return CM_EpilogueNotNeededFoldTail;
}
@@ -8529,7 +8536,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
BestMainPlan, BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
*PSE.getSE(), ResumeValues);
- LVP.attachRuntimeChecks(BestEpiPlan, Checks, HasBranchWeights);
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
// Save the status of epilogue tail-folding:
const bool IsTailFolded = BestEpiPlan.hasTailFolded();
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 7de7b3e94f4f1..6122fc57d7fee 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -22,6 +22,9 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -force-partial-aliasing-vectorization \
+; RUN: -force-target-supports-masked-memory-ops < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ALIAS-MASK
+
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is enabled
@@ -41,6 +44,9 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-INVALID-BIGER-EPILOGUE-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK-INVALID-BIGER-EPILOGUE: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
+; CHECK-ALIAS-MASK-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-ALIAS-MASK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with alias masking
+
entry:
br label %for.body
>From 8ba5bcc12aceee2ec499f902d769ef3fbeb2a089 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 9 Sep 2026 15:46:20 +0100
Subject: [PATCH 16/16] resolve review comments and add extra coverage tests
---
.../Vectorize/LoopVectorizationPlanner.h | 8 -
.../Transforms/Vectorize/LoopVectorize.cpp | 139 ++++------
.../Transforms/Vectorize/VPlanLowering.cpp | 34 ++-
.../AArch64/fold-epilogue-tail-reductions.ll | 96 ++++++-
.../AArch64/fold-epilogue-tail.ll | 241 ++++++++++++++++--
.../AArch64/partial-reduce-with-predicate.ll | 214 +++++++++++++---
...g-vectorization-fixed-order-recurrences.ll | 32 ++-
.../LoopVectorize/fold-epilogue-tail.ll | 36 +--
8 files changed, 610 insertions(+), 190 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 3399d795b04fd..aef05c515f2e3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -875,10 +875,6 @@ class LoopVectorizationPlanner {
/// The profitability analysis. Cleared after making cost based decisions.
std::unique_ptr<LoopVectorizationCostModel> CM;
- /// The profitability analysis for epilogue tail-folding.
- /// Cleared after making cost based decisions.
- std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM;
-
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
@@ -919,7 +915,6 @@ class LoopVectorizationPlanner {
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM,
- std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM,
VFSelectionContext &Config, InterleavedAccessInfo &IAI,
PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
@@ -934,9 +929,6 @@ class LoopVectorizationPlanner {
/// Destroy the cost model.
void clearCostModel();
- /// Destroy the epilogue tail-folding cost model.
- void clearEpilogueTfCM();
-
/// Build VPlans for the specified \p UserVF and \p UserIC if they are
/// non-zero or all applicable candidate VFs otherwise. If vectorization and
/// interleaving should be avoided up-front, no plans are generated.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2e8ad09f89a72..89fd7b3ea7dfe 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5561,7 +5561,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
if (!VPlans.empty() && (VPlans.front()->getSingleVF() == UserVF) &&
(UserVF.isScalar() ||
cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid())) {
- // Plan for epilogue only if we succeeded in building main loop vplan.
+ // Plan for epilogue only if we succeeded in building main loop Vplan.
// Try to plan for tail-folded epilogue if it's enabled/doable,
// otherwise plan for unpredicated epilogue:
if (!planForEpilogueTF()) {
@@ -5604,26 +5604,43 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
}
bool LoopVectorizationPlanner::planForEpilogueTF() {
- if (!EpilogueTfCM)
+ EpilogueLowering EpilogueTailLoweringStatus =
+ getEpilogueTailLowering(*CM, OrigLoop, ORE, *Legal, Config.getHints());
+ if (EpilogueTailLoweringStatus !=
+ EpilogueLowering::CM_EpilogueNotNeededFoldTail)
return false;
- assert(EpilogueTfCM->preferTailFoldedLoop() &&
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
+
+ bool UseInterleaved = TTI.enableInterleavedAccessVectorization();
+ if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
+ UseInterleaved = EnableInterleavedMemAccesses;
+
+ InterleavedAccessInfo EpilogueTfCMIAI(PSE, OrigLoop, DT, LI, Legal->getLAI(),
+ Config.OptForSize);
+ if (UseInterleaved)
+ EpilogueTfCMIAI.analyzeInterleaving(useMaskedInterleavedAccesses(TTI));
+ LoopVectorizationCostModel EpilogueTfCM(
+ EpilogueTailLoweringStatus, OrigLoop, PSE, LI, Legal, TTI, TLI, CM->AC,
+ ORE, CM->GetBFI, CM->TheFunction, EpilogueTfCMIAI, Config);
+
+ assert(EpilogueTfCM.preferTailFoldedLoop() &&
"Epilogue tail-folding is expected to be enabled");
LLVM_DEBUG(dbgs() << "LV: plan for tail-folded epilogue\n");
- EpilogueTfCM->ValuesToIgnore.insert_range(CM->ValuesToIgnore);
- EpilogueTfCM->VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
+ EpilogueTfCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
+ EpilogueTfCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
FixedScalableVFPair MaxFactors =
- EpilogueTfCM->computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
- if (!MaxFactors || !EpilogueTfCM->foldTailByMasking()) {
+ EpilogueTfCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
+ if (!MaxFactors || !EpilogueTfCM.foldTailByMasking()) {
// Cases that should not to be vectorized or tail-folded.
reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
return false;
}
- auto VPlan1 = tryToBuildVPlan1(*EpilogueTfCM);
+ auto VPlan1 = tryToBuildVPlan1(EpilogueTfCM);
// If we're here, the main loop's initial VPlan was built successfully.
// Building one for the tail-folded loop should therefore also succeed, since
@@ -5641,35 +5658,40 @@ bool LoopVectorizationPlanner::planForEpilogueTF() {
LLVM_DEBUG(
dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by "
"masking which requires masked-interleaved support.\n");
- if (EpilogueTfCM->InterleaveInfo.invalidateGroups())
+ if (EpilogueTfCM.InterleaveInfo.invalidateGroups())
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
- EpilogueTfCM->invalidateCostModelingDecisions();
+ EpilogueTfCM.invalidateCostModelingDecisions();
}
Legal->prepareToFoldTailByMasking();
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- EpilogueTfCM->collectNonVectorizedAndSetWideningDecisions(
+ EpilogueTfCM.collectNonVectorizedAndSetWideningDecisions(
EpilogueVectorizationForceVF);
size_t NumPlansBefore = VPlans.size();
buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
- EpilogueVectorizationForceVF, *EpilogueTfCM);
+ EpilogueVectorizationForceVF, EpilogueTfCM);
- // Check that a vplan is successfully built:
- if (VPlans.size() == NumPlansBefore ||
- VPlans.back()->getSingleVF() != EpilogueVectorizationForceVF ||
+ // Check that a Vplan is successfully built:
+ if (VPlans.size() == NumPlansBefore) {
+ reportVectorizationInfo("Failed to build tail-folded epilogue VPlan",
+ "InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ return false;
+ }
+ if (VPlans.back()->getSingleVF() != EpilogueVectorizationForceVF ||
!VPlans.back()->hasTailFolded()) {
reportVectorizationInfo(
"Failed to build a valid tail-folded epilogue VPlan",
"InvalidTailFoldedEpilogue", ORE, OrigLoop);
+ VPlans.pop_back();
return false;
}
if (!cost(*VPlans.back(), EpilogueVectorizationForceVF, /*RU=*/nullptr,
- *EpilogueTfCM)
+ EpilogueTfCM)
.isValid()) {
VPlans.pop_back();
reportVectorizationInfo("This case of epilogue loop can't be tail-folded "
@@ -6026,20 +6048,16 @@ LoopVectorizationPlanner::computeBestVF() {
LoopVectorizationPlanner::LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
- std::unique_ptr<LoopVectorizationCostModel> CM,
- std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM,
- VFSelectionContext &Config, InterleavedAccessInfo &IAI,
- PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE)
+ std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
+ InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
+ OptimizationRemarkEmitter *ORE)
: OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(std::move(CM)), EpilogueTfCM(std::move(EpilogueTfCM)), Config(Config),
- IAI(IAI), PSE(PSE), ORE(ORE) {}
+ CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
-void LoopVectorizationPlanner::clearEpilogueTfCM() { EpilogueTfCM.reset(); }
-
DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT,
@@ -6067,9 +6085,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan, BestVF, VScale);
}
- const bool IsTailFolded = BestVPlan.hasTailFolded();
if (vputils::findIncomingAliasMask(BestVPlan)) {
- assert(IsTailFolded && "Expected tail folding to be enabled");
+ assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
*Legal->getRuntimePointerChecking()->getDiffChecks(),
HasBranchWeights);
@@ -6109,6 +6126,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
// Regions are dissolved after optimizing for VF and UF, which completely
// removes unneeded loop regions first.
+ const bool HasTailFolded = BestVPlan.hasTailFolded();
RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
// Expand BranchOnTwoConds after dissolution, when latch has direct access to
// its successors.
@@ -6128,7 +6146,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
"loops not exiting via the latch without required epilogue?");
RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
- VectorPH, IsTailFolded, RequiresScalarEpilogue,
+ VectorPH, HasTailFolded, RequiresScalarEpilogue,
&BestVPlan.getVFxUF(), MaxRuntimeStep);
RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
BestVF);
@@ -8004,39 +8022,6 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
for (PHINode &Phi : make_early_inc_range(VecEpiloguePreHeader->phis()))
if (Phi.use_empty())
Phi.eraseFromParent();
-
- if (IsEpilogueTfEnabled) {
- // The epilogue vector loop is tail-folded, so it can safely handle
- // any remaining iterations, including zero, via masking.
- // vec.epilog.iter.check's own min-iters check was therefore built with a
- // compile-time-known-false condition (see
- // addMinimumVectorEpilogueIterationCheck) that never needs to bail out to
- // a scalar remainder. Fold it into an unconditional branch into the
- // vector epilogue preheader.
- auto *Br =
- cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator());
- [[maybe_unused]] auto *CondC = dyn_cast<ConstantInt>(Br->getCondition());
- assert(CondC && CondC->isZero() &&
- "expected vec.epilog.iter.check's branch condition to be a "
- "compile-time false constant when the epilogue is tail-folded");
- BasicBlock *DeadSucc = Br->getSuccessor(0);
- UncondBrInst::Create(VecEpiloguePreHeader, Br->getIterator());
- Br->eraseFromParent();
- DTU.applyUpdates(
- {{DominatorTree::Delete, VecEpilogueIterationCountCheck, DeadSucc}});
-
- if (!SCEVCheckBlock && !MemCheckBlock) {
- // Delete the scalar loop as it's dead right now.
- assert(pred_empty(ScalarPH) &&
- "scalar preheader should have no predecessors left");
- SmallVector<BasicBlock *> Blocks(L->block_begin(), L->block_end());
- Blocks.push_back(ScalarPH);
- LI->erase(L);
- for (auto *BB : Blocks)
- LI->removeBlock(BB);
- DeleteDeadBlocks(Blocks, &DTU);
- }
- }
}
bool LoopVectorizePass::processLoop(Loop *L) {
@@ -8220,32 +8205,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Use the cost model.
VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
OptForSize);
- auto CM = std::make_unique<LoopVectorizationCostModel>(
- SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config);
-
- // Setup the epilogue tail-folding CM. Only built when tail-folding the
- // epilogue is actually a candidate, to avoid the cost of an extra
- // InterleavedAccessInfo scan and LoopVectorizationCostModel construction
- // for the common case where this (experimental, off-by-default) feature
- // isn't in use.
- EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(*CM, L, ORE, LVL, Hints);
- std::optional<InterleavedAccessInfo> EpilogueTfCMIAI;
- std::unique_ptr<LoopVectorizationCostModel> EpilogueTfCM;
- if (EpilogueTailLoweringStatus ==
- EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
- LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
- EpilogueTfCMIAI.emplace(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
- if (UseInterleaved)
- EpilogueTfCMIAI->analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
- EpilogueTfCM = std::make_unique<LoopVectorizationCostModel>(
- EpilogueTailLoweringStatus, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
- GetBFI, F, *EpilogueTfCMIAI, Config);
- }
-
// Use the planner for vectorization.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, std::move(CM),
- std::move(EpilogueTfCM), Config, IAI, PSE, ORE);
+ LoopVectorizationPlanner LVP(
+ L, LI, DT, TLI, *TTI, &LVL,
+ std::make_unique<LoopVectorizationCostModel>(
+ SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
+ Config, IAI, PSE, ORE);
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
@@ -8259,10 +8224,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Plan how to best vectorize.
LVP.plan(UserVF, UserIC);
- // Right now, after planning, the epilogue tail-folding CM is not needed
- // anymore. Clear it.
- LVP.clearEpilogueTfCM();
-
auto [VF, BestPlanPtr] = LVP.computeBestVF();
unsigned IC = 1;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 08cc0f57acce6..8633ec04d0023 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -49,29 +49,30 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
VPIRValue *StartValue = nullptr;
// VPWidenCanonicalIVRecipe is either a direct user of CanonicalIV or
// Add (CanonicalIV, resumeValue) (like the case for tail-folded epilogue).
- for (auto *User : LoopRegion->getCanonicalIV()->users()) {
+ auto *IV = LoopRegion->getCanonicalIV();
+ for (auto *User : IV->users()) {
if (isa<VPWidenCanonicalIVRecipe>(User)) {
WideCanIV = cast<VPWidenCanonicalIVRecipe>(User);
StartValue = Plan.getZero(WideCanIV->getScalarType());
break;
}
- if (isa<VPInstruction>(User)) {
- auto *UserInstr = cast<VPInstruction>(User);
- auto *It = find_if(UserInstr->users(), IsaPred<VPWidenCanonicalIVRecipe>);
- if (It != UserInstr->user_end()) {
+ if (auto *UI = dyn_cast<VPInstruction>(User)) {
+ auto *It = find_if(UI->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ if (It != UI->user_end()) {
WideCanIV = cast<VPWidenCanonicalIVRecipe>(*It);
- match(UserInstr, m_Add(m_VPValue(), m_VPIRValue(StartValue)));
- assert(StartValue &&
- "WIDEN-CANONICAL-INDUCTION is only expected to be reached "
- "through the canonical IV directly, or through a single 'add "
- "CanonicalIV, StartValue' introduced for epilogue-loop resume "
- "values; found a different pattern here");
- if (!StartValue)
+ if (!match(UI, m_c_Add(m_Specific(IV), m_VPIRValue(StartValue))) ||
+ !StartValue) {
+ assert(StartValue &&
+ "WIDEN-CANONICAL-INDUCTION is only expected to be reached "
+ "through the canonical IV directly, or through a single 'add "
+ "CanonicalIV, StartValue' introduced for epilogue-loop resume "
+ "values; found a different pattern here");
return;
+ }
}
}
}
- if (!WideCanIV)
+ if (!WideCanIV || !StartValue)
return;
Type *CanIVTy = WideCanIV->getScalarType();
@@ -579,7 +580,12 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
}
if (auto *WideCanIV = dyn_cast<VPWidenCanonicalIVRecipe>(&R)) {
- VPValue *CanIV = WideCanIV->getCanonicalIV();
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ if (!LoopRegion)
+ continue;
+ VPValue *CanIV = LoopRegion->getCanonicalIV();
+ if (!CanIV)
+ continue;
Type *CanIVTy = CanIV->getScalarType();
VPValue *Step = WideCanIV->getStepValue();
if (!Step) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
index 1c9c05884b7d4..4323230e2b525 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail-reductions.ll
@@ -41,7 +41,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -64,8 +64,19 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 1
+; CHECK-NEXT: [[ADD]] = add i32 [[LOAD]], [[RED]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[ADD_LCSSA]]
;
; CHECK-VS-LABEL: define i32 @add_redc(
@@ -104,7 +115,7 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -129,8 +140,19 @@ define i32 @add_redc(ptr %src, i64 %n) {
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP17]])
; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-VS-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 1
+; CHECK-VS-NEXT: [[ADD]] = add i32 [[LOAD]], [[RED]]
+; CHECK-VS-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-VS-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-VS-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP20]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[ADD_LCSSA]]
;
entry:
@@ -183,7 +205,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -207,8 +229,19 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> [[TMP10]])
; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MAX:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 1
+; CHECK-NEXT: [[MAX]] = call i32 @llvm.umax.i32(i32 [[LOAD]], i32 [[RED]])
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[MAX]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[MAX_LCSSA]]
;
; CHECK-VS-LABEL: define i32 @max_redc(
@@ -247,7 +280,7 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -273,8 +306,19 @@ define i32 @max_redc(ptr %src, i64 %n) {
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.umax.nxv8i32(<vscale x 8 x i32> [[TMP16]])
; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MAX:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-VS-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 1
+; CHECK-VS-NEXT: [[MAX]] = call i32 @llvm.umax.i32(i32 [[LOAD]], i32 [[RED]])
+; CHECK-VS-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-VS-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-VS-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[MAX_LCSSA:%.*]] = phi i32 [ [[MAX]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[MAX_LCSSA]]
;
entry:
@@ -455,7 +499,7 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -483,8 +527,20 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP18]]
; CHECK-NEXT: [[RDX_SELECT7:%.*]] = select i1 [[TMP19]], i32 1, i32 0
; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SELECT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[ICMP:%.*]] = icmp eq i8 [[LOAD]], 0
+; CHECK-NEXT: [[SELECT]] = select i1 [[ICMP]], i32 1, i32 [[RED]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[SELECT_LCSSA]]
;
; CHECK-VS-LABEL: define i32 @any-of(
@@ -527,7 +583,7 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -557,8 +613,20 @@ define i32 @any-of(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP24]]
; CHECK-VS-NEXT: [[RDX_SELECT7:%.*]] = select i1 [[TMP25]], i32 1, i32 0
; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SELECT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-VS-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-VS-NEXT: [[ICMP:%.*]] = icmp eq i8 [[LOAD]], 0
+; CHECK-VS-NEXT: [[SELECT]] = select i1 [[ICMP]], i32 1, i32 [[RED]]
+; CHECK-VS-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-VS-NEXT: [[ICMP3:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-VS-NEXT: br i1 [[ICMP3]], label %[[EXIT]], label %[[LOOP]]
; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[SELECT_LCSSA:%.*]] = phi i32 [ [[SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_SELECT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[SELECT_LCSSA]]
;
entry:
@@ -621,7 +689,7 @@ define i64 @arg_min_first_index(ptr %arr, i64 %n, i64 %start) {
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -729,7 +797,7 @@ define i64 @arg_min_first_index(ptr %arr, i64 %n, i64 %start) {
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-VS-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
-; CHECK-VS-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-VS-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
index 3cee861a822b4..a610939954df2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-epilogue-tail.ll
@@ -1,12 +1,31 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; REQUIRES: asserts
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 -epilogue-vectorization-force-VF=8 -mattr=+sve %s | FileCheck %s
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -mattr=+sve \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 \
+; RUN: -epilogue-vectorization-force-VF=8 %s | FileCheck %s
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width="vscale x 16" -epilogue-vectorization-force-VF="vscale x 8" -mattr=+sve %s | FileCheck %s --check-prefix=CHECK-VS
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -mattr=+sve \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width="vscale x 16" \
+; RUN: -epilogue-vectorization-force-VF="vscale x 8" %s | FileCheck %s \
+; RUN: --check-prefix=CHECK-VS
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail -debug-only=loop-vectorize,vectorutils --disable-output \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail --disable-output \
+; RUN: -force-vector-width=4 -epilogue-vectorization-force-VF="vscale x 2" \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s \
+; RUN: --check-prefix=CHECK-INVALID-COSTS
+
+; RUN: opt -p loop-vectorize -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -force-vector-width=8 -epilogue-vectorization-force-VF=4 \
+; RUN: -force-target-supports-masked-memory-ops -S %s | FileCheck %s \
+; RUN: --check-prefix=CHECK-INDUCTION
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize,vectorutils \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail --disable-output \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INVALIDATE-INTERLEAVE
target triple = "aarch64-linux-gnu"
@@ -38,7 +57,7 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
@@ -57,6 +76,15 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[VAL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -91,7 +119,7 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -112,6 +140,15 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i32 %val) {
; CHECK-VS-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-VS: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-VS: [[FOR_BODY]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VS-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VS-NEXT: store i32 [[VAL]], ptr [[ARRAYIDX]], align 4
+; CHECK-VS-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-VS-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT]]
; CHECK-VS: [[EXIT]]:
; CHECK-VS-NEXT: ret void
;
@@ -156,7 +193,7 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[VEC_EPILOG_RESUME_VAL]], i64 [[N]])
@@ -177,8 +214,17 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[FOR_END]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[FOR_END]], label %[[LOOP]]
; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[LOAD]], %[[LOOP]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[LOAD_LCSSA]]
;
; CHECK-VS-LABEL: define i32 @live-out(
@@ -213,7 +259,7 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
@@ -236,8 +282,17 @@ define i32 @live-out(ptr %src, i64 %n) {
; CHECK-VS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-VS-NEXT: [[TMP17:%.*]] = extractelement <vscale x 8 x i32> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-VS-NEXT: br label %[[FOR_END]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-VS-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-VS-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VS-NEXT: br i1 [[EC]], label %[[FOR_END]], label %[[LOOP]]
; CHECK-VS: [[FOR_END]]:
-; CHECK-VS-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[LOAD_LCSSA:%.*]] = phi i32 [ [[LOAD]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[LOAD_LCSSA]]
;
entry:
@@ -282,7 +337,7 @@ define i32 @live_out_recurrence(ptr %A, i64 %n) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -307,8 +362,18 @@ define i32 @live_out_recurrence(ptr %A, i64 %n) {
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP7]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[FOR:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[FOR_LCSSA]]
;
; CHECK-VS-LABEL: define i32 @live_out_recurrence(
@@ -347,7 +412,7 @@ define i32 @live_out_recurrence(ptr %A, i64 %n) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -377,8 +442,18 @@ define i32 @live_out_recurrence(ptr %A, i64 %n) {
; CHECK-VS-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-VS-NEXT: [[TMP23:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-VS-NEXT: br label %[[EXIT]]
+; CHECK-VS: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-VS-NEXT: br label %[[LOOP:.*]]
+; CHECK-VS: [[LOOP]]:
+; CHECK-VS-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[FOR:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-VS-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VS-NEXT: [[L]] = load i32, ptr [[GEP]], align 4
+; CHECK-VS-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VS-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VS-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
; CHECK-VS: [[EXIT]]:
-; CHECK-VS-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VS-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-VS-NEXT: ret i32 [[FOR_LCSSA]]
;
entry:
@@ -441,7 +516,7 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[VAL]], i64 0
@@ -527,7 +602,7 @@ define void @reversed-loop(ptr %A, i32 %n, i32 %val) {
; CHECK-VS-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-VS-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-VS: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-VS-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-VS-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
; CHECK-VS: [[VEC_EPILOG_PH]]:
; CHECK-VS-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-VS-NEXT: [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
@@ -583,6 +658,140 @@ exit:
ret void
}
+define void @alloca(ptr %vla, i64 %N) {
+; CHECK-INVALID-COSTS-LABEL: Checking a loop in 'alloca'
+; CHECK-INVALID-COSTS: LV: epilogue tail-folding is enabled
+; CHECK-INVALID-COSTS: remark: <unknown>:0:0: This case of epilogue loop can't be tail-folded - Invalid costs
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+ %alloca = alloca i32, align 16
+ %arrayidx = getelementptr inbounds ptr, ptr %vla, i64 %iv
+ store ptr %alloca, ptr %arrayidx, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ call void @foo(ptr nonnull %vla)
+ ret void
+}
+declare void @foo(ptr)
+
+define i64 @find_last_offset_wide_canonical_iv(ptr %A, i64 %n) {
+; CHECK-INDUCTION-LABEL: define i64 @find_last_offset_wide_canonical_iv(
+; CHECK-INDUCTION-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-INDUCTION-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-INDUCTION-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-INDUCTION-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-INDUCTION: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-INDUCTION-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-INDUCTION-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-INDUCTION: [[VECTOR_PH]]:
+; CHECK-INDUCTION-NEXT: [[TMP0:%.*]] = and i64 [[N]], 15
+; CHECK-INDUCTION-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-INDUCTION-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INDUCTION: [[VECTOR_BODY]]:
+; CHECK-INDUCTION-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_PHI:%.*]] = phi <8 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-INDUCTION-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-INDUCTION-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 8
+; CHECK-INDUCTION-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
+; CHECK-INDUCTION-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; CHECK-INDUCTION-NEXT: [[TMP3:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 11)
+; CHECK-INDUCTION-NEXT: [[TMP22:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD3]], splat (i32 11)
+; CHECK-INDUCTION-NEXT: [[TMP12]] = select <8 x i1> [[TMP3]], <8 x i64> [[VEC_IND]], <8 x i64> [[VEC_PHI]]
+; CHECK-INDUCTION-NEXT: [[TMP15]] = select <8 x i1> [[TMP22]], <8 x i64> [[STEP_ADD]], <8 x i64> [[VEC_PHI2]]
+; CHECK-INDUCTION-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INDUCTION-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i64> [[STEP_ADD]], splat (i64 8)
+; CHECK-INDUCTION-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INDUCTION-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; CHECK-INDUCTION: [[MIDDLE_BLOCK]]:
+; CHECK-INDUCTION-NEXT: [[RDX_MINMAX:%.*]] = call <8 x i64> @llvm.smax.v8i64(<8 x i64> [[TMP12]], <8 x i64> [[TMP15]])
+; CHECK-INDUCTION-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> [[RDX_MINMAX]])
+; CHECK-INDUCTION-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-INDUCTION-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-INDUCTION-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INDUCTION-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-INDUCTION: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-INDUCTION-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
+; CHECK-INDUCTION: [[VEC_EPILOG_PH]]:
+; CHECK-INDUCTION-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-INDUCTION-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-INDUCTION-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], -1
+; CHECK-INDUCTION-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
+; CHECK-INDUCTION-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-INDUCTION-NEXT: [[TMP10:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-INDUCTION-NEXT: [[N_VEC2:%.*]] = sub i64 [[N_RND_UP]], [[TMP10]]
+; CHECK-INDUCTION-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-INDUCTION-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT7]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-INDUCTION-NEXT: [[INDUCTION:%.*]] = add nuw <4 x i64> [[BROADCAST_SPLAT8]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-INDUCTION-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-INDUCTION: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-INDUCTION-NEXT: [[TMP11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[TMP17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_IND10:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_PHI11:%.*]] = phi <4 x i64> [ [[BROADCAST_SPLAT6]], %[[VEC_EPILOG_PH]] ], [ [[TMP23:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[VEC_IND12:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-INDUCTION-NEXT: [[TMP14:%.*]] = icmp ule <4 x i64> [[VEC_IND12]], [[BROADCAST_SPLAT]]
+; CHECK-INDUCTION-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
+; CHECK-INDUCTION-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP13]], <4 x i1> [[TMP14]], <4 x i32> poison)
+; CHECK-INDUCTION-NEXT: [[TMP16:%.*]] = icmp eq <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 11)
+; CHECK-INDUCTION-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP14]], <4 x i1> [[TMP16]], <4 x i1> zeroinitializer
+; CHECK-INDUCTION-NEXT: [[TMP23]] = select <4 x i1> [[TMP24]], <4 x i64> [[VEC_IND10]], <4 x i64> [[VEC_PHI11]]
+; CHECK-INDUCTION-NEXT: [[TMP17]] = add i64 [[TMP11]], 4
+; CHECK-INDUCTION-NEXT: [[VEC_IND_NEXT14]] = add nuw nsw <4 x i64> [[VEC_IND10]], splat (i64 4)
+; CHECK-INDUCTION-NEXT: [[VEC_IND_NEXT15]] = add nuw <4 x i64> [[VEC_IND12]], splat (i64 4)
+; CHECK-INDUCTION-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], [[N_VEC2]]
+; CHECK-INDUCTION-NEXT: br i1 [[TMP18]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
+; CHECK-INDUCTION: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-INDUCTION-NEXT: [[TMP19:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP23]])
+; CHECK-INDUCTION-NEXT: [[TMP20:%.*]] = icmp ne i64 [[TMP19]], -9223372036854775808
+; CHECK-INDUCTION-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[TMP19]], i64 -1
+; CHECK-INDUCTION-NEXT: br label %[[EXIT]]
+; CHECK-INDUCTION: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-INDUCTION-NEXT: br label %[[LOOP:.*]]
+; CHECK-INDUCTION: [[LOOP]]:
+; CHECK-INDUCTION-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-INDUCTION-NEXT: [[RED:%.*]] = phi i64 [ -1, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-INDUCTION-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-INDUCTION-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-INDUCTION-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 11
+; CHECK-INDUCTION-NEXT: [[SEL]] = select i1 [[C]], i64 [[IV]], i64 [[RED]]
+; CHECK-INDUCTION-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INDUCTION-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INDUCTION-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-INDUCTION: [[EXIT]]:
+; CHECK-INDUCTION-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-INDUCTION-NEXT: ret i64 [[SEL_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %c = icmp eq i32 %l, 11
+ %sel = select i1 %c, i64 %iv, i64 %red
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i64 %sel
+}
+
define i64 @test_no_masked_interleave_support(i64 %y, i32 %n) {
; CHECK-INVALIDATE-INTERLEAVE-LABEL: Checking a loop in 'test_no_masked_interleave_support'
; CHECK-INVALIDATE-INTERLEAVE: LV: epilogue tail-folding is enabled
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index de28bb59b6bab..8196fcdc6dc98 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -115,14 +115,14 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -146,12 +146,32 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
;
entry:
@@ -292,14 +312,14 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -323,12 +343,32 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
;
entry:
@@ -489,14 +529,14 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP16]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -524,12 +564,36 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT14]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE13]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
;
entry:
@@ -696,7 +760,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP16]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
@@ -704,7 +768,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -731,13 +795,37 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT14]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP30:%.*]] = xor i1 [[TMP29]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE13]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP32:%.*]] = sub i32 [[BC_MERGE_RDX]], [[TMP31]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP32]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ [[TMP32]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
;
entry:
@@ -903,14 +991,14 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE9]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE5]], <16 x i32> [[TMP14]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE9]], [[PARTIAL_REDUCE8]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -939,12 +1027,36 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT17]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP27]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE16]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_2_LCSSA]]
;
entry:
@@ -1110,14 +1222,14 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE9]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE4]], <16 x i32> [[TMP14]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE9]], [[PARTIAL_REDUCE8]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -1146,12 +1258,36 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT17]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP27]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP28]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE16]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[FOR_BODY]] ], [ [[ADD]], %[[IF_THEN]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_2_LCSSA]]
;
entry:
@@ -1263,7 +1399,7 @@ define i32 @pred_reduction_incoming_1(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
-; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
@@ -1304,14 +1440,14 @@ define i32 @pred_reduction_incoming_1(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[PARTIAL_REDUCE5]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -1335,12 +1471,32 @@ define i32 @pred_reduction_incoming_1(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT11]], i64 [[N]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TMP21]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE10]])
; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLD-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[IF_THEN:.*]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV:%.*]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM:%.*]], [[CONV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br label %[[FOR_INC:.*]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_BODY]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM]] = phi i32 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN]]
+; CHECK-TAILFOLD-EPILOGUE: [[FOR_INC]]:
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]]
; CHECK-TAILFOLD-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLD-EPILOGUE-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLD-EPILOGUE-NEXT: ret i32 [[SUM_1_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
index b2d5b6a3322ea..e1086979859cf 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
@@ -94,7 +94,7 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
@@ -121,6 +121,18 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[LOOP:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[LOOP]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR:%.*]] = phi i64 [ 99, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
; CHECK-TAILFOLDED-EPILOGUE: [[EXIT]]:
; CHECK-TAILFOLDED-EPILOGUE-NEXT: ret void
;
@@ -346,7 +358,7 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[VEC_EPILOG_PH]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VEC_EPILOG_PH]]
; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_PH]]:
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -382,9 +394,21 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[EXIT]]
+; CHECK-TAILFOLDED-EPILOGUE: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br label %[[LOOP:.*]]
+; CHECK-TAILFOLDED-EPILOGUE: [[LOOP]]:
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR:%.*]] = phi i64 [ 99, %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
; CHECK-TAILFOLDED-EPILOGUE: [[EXIT]]:
-; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[TMP11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
-; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[TMP12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ], [ [[TMP11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
; CHECK-TAILFOLDED-EPILOGUE-NEXT: ret i64 [[RES]]
;
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 6122fc57d7fee..d1bd3b2d1b63e 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -33,10 +33,10 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
;
; CHECK-NO-FORCED-MAIN-VF-LABEL: Checking a loop in 'test_epilogue_tf'
-; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
+; CHECK-NO-FORCED-MAIN-VF-NOT: LV: epilogue tail-folding is enabled
; CHECK-NO-FORCED-EPILOGUE-VF-LABEL: Checking a loop in 'test_epilogue_tf'
-; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
+; CHECK-NO-FORCED-EPILOGUE-VF-NOT: LV: epilogue tail-folding is enabled
;
; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
@@ -152,26 +152,31 @@ for.end:
ret i32 0
}
-define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
+define i64 @early_exit(ptr dereferenceable(1024) align 8 %src, i1 %cond) {
; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'early_exit'
; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
;
entry:
- br label %for.body
+ br label %loop.header
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
- %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
- %val = load i8, ptr %arrayidx, align 1
- %exitcond = icmp eq i8 %val, %find
- br i1 %exitcond, label %exit, label %cont
+loop.header:
+ %iv = phi i64 [ %iv.next, %latch ], [ 0, %entry ]
+ %gep = getelementptr inbounds double, ptr %src, i64 %iv
+ %val = load double, ptr %gep, align 8
+ %neg = fneg double %val
+ %c.1 = fcmp une double %neg, 10.0
+ br i1 %c.1, label %latch, label %early.exit
+
+latch:
+ %iv.next = add nuw i64 %iv, 1
+ %exit.cond = icmp eq i64 %iv.next, 127
+ br i1 %exit.cond, label %exit, label %loop.header
+
+early.exit:
+ ret i64 %iv
-cont:
- %iv.next = add nuw nsw i64 %iv, 1
- %contcond = icmp ne i64 %iv.next, %n
- br i1 %contcond, label %for.body, label %exit
exit:
- ret i1 %exitcond
+ ret i64 10
}
; For this function, the check line is not related to epilogue tail-folding, but when vectorizing this case gets supported,
@@ -204,7 +209,6 @@ exit:
define void @test_outer_loop(ptr %A, i64 %m) {
; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
-; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
; CHECK-OUTER-LOOP-NOT: LV: epilogue tail-folding is enabled
;
entry:
More information about the llvm-commits
mailing list