[llvm-branch-commits] [llvm] Patch 4: [LV][NFC]: Filter between vplans by tail-folding status (PR #207165)
Hassnaa Hamdi via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 2 05:18:47 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/207165
>From 82c7a98d82983dabd2a492e0264e096b31ed1667 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 1 Jul 2026 22:18:09 +0100
Subject: [PATCH] Patch 4: [LV][NFC]: Filter between vplans by tail-folding
status
---
.../Vectorize/LoopVectorizationPlanner.h | 7 +---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++++++----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 37 +++++++++++++++++--
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +
4 files changed, 53 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 249b00a6da196..20dde28e75b4e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -872,7 +872,7 @@ class LoopVectorizationPlanner {
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF) const;
+ VPlan &getPlanFor(ElementCount VF, bool TF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
@@ -916,10 +916,7 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF) const {
- return any_of(VPlans,
- [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
- }
+ bool hasPlanWithVF(ElementCount VF, bool TF) const;
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9f5025ff30642..c10d9af1b126a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3147,6 +3147,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
+ if (!Plan->isCompatibleWithTF(CM.foldTailByMasking()))
+ continue;
for (ElementCount VF : Plan->vectorFactors()) {
// The VPlan-based cost model is designed for computing vector cost.
// Querying VPlan-based cost model with a scarlar VF will cause some
@@ -3490,8 +3492,9 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF);
- if (hasPlanWithVF(ForcedEC)) {
- std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC).duplicate());
+ if (hasPlanWithVF(ForcedEC, CM.foldTailByMasking())) {
+ std::unique_ptr<VPlan> Clone(
+ getPlanFor(ForcedEC, CM.foldTailByMasking()).duplicate());
Clone->setVF(ForcedEC);
return Clone;
}
@@ -3582,10 +3585,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width))
+ if (!hasPlanWithVF(NextVF.Width, CM.foldTailByMasking()))
continue;
- VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ VPlan &CurrentPlan = getPlanFor(NextVF.Width, CM.foldTailByMasking());
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
// vectors) or > the VF of the main loop (fixed vectors).
@@ -5835,13 +5838,15 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF(
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
+ assert((FirstPlan.hasScalarVFOnly() ||
+ hasPlanWithVF(UserVF, CM.foldTailByMasking()) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && EpilogueVectorizationForceVF > 1) {
+ if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) &&
+ EpilogueVectorizationForceVF > 1) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
assert(VPlans[0]->getSingleVF() ==
ElementCount::getFixed(EpilogueVectorizationForceVF) &&
@@ -5882,6 +5887,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF(
VPlan *PlanForBestVF = &FirstPlan;
for (auto &P : VPlans) {
+ if (!P->isCompatibleWithTF(CM.foldTailByMasking()))
+ continue;
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
P->vectorFactors().end());
@@ -8158,7 +8165,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
CM.maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost, CM);
@@ -8220,7 +8227,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) &&
+ UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9c0bec2350c97..b5a776d000b8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1308,6 +1308,28 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
+bool VPlan::isCompatibleWithTF(bool TF) {
+ bool HasHeaderMask = (vputils::findHeaderMask(*this) != nullptr);
+ bool NoTail = !hasScalarTail();
+ // Check if TC is not divisible by the VF to make sure that the plan doesn't
+ // have a tail because the tail is folded not because there is no remaining
+ // iterations. For scalable VFs, divisibility cannot be determined at
+ // compile-time, so conservatively keep TCNotDivisibleByVF = true.
+ bool TCNotDivisibleByVF = true;
+ if (auto *ConstTC = dyn_cast<VPConstantInt>(getTripCount())) {
+ const APInt &TC =
+ cast<ConstantInt>(ConstTC->getLiveInIRValue())->getValue();
+ if (none_of(vectorFactors(),
+ [](ElementCount VF) { return VF.isScalable(); }))
+ TCNotDivisibleByVF = any_of(vectorFactors(), [&TC](ElementCount VF) {
+ APInt VFVal(TC.getBitWidth(), VF.getFixedValue());
+ return TC.urem(VFVal) != 0;
+ });
+ }
+ NoTail &= TCNotDivisibleByVF;
+ return (TF == (HasHeaderMask | NoTail));
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1655,19 +1677,26 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange(
return PredicateAtRangeStart;
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
assert(count_if(VPlans,
- [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
- 1 &&
+ [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ }) == 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF))
+ if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
+bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
+ return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ });
+}
+
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eaf9d1433aff7..6cb238af9804e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5155,6 +5155,8 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
+ bool isCompatibleWithTF(bool TF);
+
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
More information about the llvm-branch-commits
mailing list