[llvm-branch-commits] [llvm] Patch 4: [LV][NFC]: Filter between vplans by tail-folding status (PR #207165)
Hassnaa Hamdi via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Jul 8 05:59:30 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/207165
>From 8abccc81a7555d67a0261df4121b501d979b4db3 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 1 Jul 2026 22:18:09 +0100
Subject: [PATCH] Patch 4: [LV][NFC]: Filter between vplans by tail-folding
status
---
.../Vectorize/LoopVectorizationPlanner.h | 7 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 34 ++++++++++++++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +++++
4 files changed, 56 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4829506b3aa75..b4220d284b9ae 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -883,7 +883,7 @@ class LoopVectorizationPlanner {
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
- VPlan &getPlanFor(ElementCount VF) const;
+ VPlan &getPlanFor(ElementCount VF, bool TF) const;
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
@@ -927,10 +927,7 @@ class LoopVectorizationPlanner {
/// Look through the existing plans and return true if we have one with
/// vectorization factor \p VF.
- bool hasPlanWithVF(ElementCount VF) const {
- return any_of(VPlans,
- [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
- }
+ bool hasPlanWithVF(ElementCount VF, bool TF) const;
/// Test a \p Predicate on a \p Range of VF's. Return the value of applying
/// \p Predicate on Range.Start, possibly decreasing Range.End such that the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6901454770479..3590353932540 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3490,8 +3490,9 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
ElementCount ForcedEC = ElementCount::getFixed(EpilogueVectorizationForceVF);
- if (hasPlanWithVF(ForcedEC)) {
- std::unique_ptr<VPlan> Clone(getPlanFor(ForcedEC).duplicate());
+ if (hasPlanWithVF(ForcedEC, CM.foldTailByMasking())) {
+ std::unique_ptr<VPlan> Clone(
+ getPlanFor(ForcedEC, CM.foldTailByMasking()).duplicate());
Clone->setVF(ForcedEC);
return Clone;
}
@@ -3582,10 +3583,10 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan *BestPlan = nullptr;
for (auto &NextVF : ProfitableVFs) {
// Skip candidate VFs without a corresponding VPlan.
- if (!hasPlanWithVF(NextVF.Width))
+ if (!hasPlanWithVF(NextVF.Width, CM.foldTailByMasking()))
continue;
- VPlan &CurrentPlan = getPlanFor(NextVF.Width);
+ VPlan &CurrentPlan = getPlanFor(NextVF.Width, CM.foldTailByMasking());
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
// Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
// vectors) or > the VF of the main loop (fixed vectors).
@@ -5831,13 +5832,15 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF(
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
- assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
+ assert((FirstPlan.hasScalarVFOnly() ||
+ hasPlanWithVF(UserVF, CM.foldTailByMasking()) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && EpilogueVectorizationForceVF > 1) {
+ if (hasPlanWithVF(UserVF, CM.foldTailByMasking()) &&
+ EpilogueVectorizationForceVF > 1) {
assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
assert(VPlans[0]->getSingleVF() ==
ElementCount::getFixed(EpilogueVectorizationForceVF) &&
@@ -5908,6 +5911,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF(
continue;
}
+ assert(P->isCompatibleWithTF(CM.foldTailByMasking()) &&
+ "All vplans must be compatible with the CM");
InstructionCost Cost =
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, CM);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
@@ -5921,8 +5926,8 @@ std::pair<VectorizationFactor, VPlan *> LoopVectorizationPlanner::computeBestVF(
if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
ProfitableVFs.push_back(CurrentFactor);
- // Get the costs for the EpilogueTailFoldingCM:
if (EpilogueTailFoldingCM) {
+ // Get the costs for the EpilogueTailFoldingCM:
LLVM_DEBUG(dbgs() << "LV: Predicated CM, calculate costs for VF: " << VF
<< "\n");
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr,
@@ -8133,7 +8138,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
CM.maskPartialAliasing());
- if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
+ if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking())) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost, CM);
@@ -8195,7 +8200,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"Ignoring user-specified interleave count due to possibly "
"unsafe dependencies in the loop."};
InterleaveLoop = false;
- } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
+ } else if (!LVP.hasPlanWithVF(VF.Width, CM.foldTailByMasking()) &&
+ UserIC > 1) {
// Tell the user interleaving was avoided up-front, despite being explicitly
// requested.
LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 292e3e85da9d5..bf93d8000cad9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1310,6 +1310,25 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
return VPIRBB;
}
+bool VPlan::isCompatibleWithTF(bool TF) {
+ bool HasHeaderMask = (vputils::findHeaderMask(*this) != nullptr);
+ bool NoTail = !hasScalarTail();
+ // Check if TC is not divisible by the VF to make sure that the vplan doesn't
+ // have a tail because the tail is folded not because there is no remaining
+ // iterations.
+ bool TCNotDivisibleByVF = true;
+ if (auto *ConstTC = dyn_cast<VPConstantInt>(getTripCount())) {
+ const APInt &TC =
+ cast<ConstantInt>(ConstTC->getLiveInIRValue())->getValue();
+ TCNotDivisibleByVF = any_of(vectorFactors(), [&TC](ElementCount VF) {
+ APInt VFVal(TC.getBitWidth(), VF.getKnownMinValue());
+ return TC.urem(VFVal) != 0;
+ });
+ }
+ NoTail &= TCNotDivisibleByVF;
+ return (TF == (HasHeaderMask | NoTail));
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1678,19 +1697,26 @@ VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
-VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF, bool TF) const {
assert(count_if(VPlans,
- [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
- 1 &&
+ [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ }) == 1 &&
"Multiple VPlans for VF.");
for (const VPlanPtr &Plan : VPlans) {
- if (Plan->hasVF(VF))
+ if (Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF))
return *Plan.get();
}
llvm_unreachable("No plan found!");
}
+bool LoopVectorizationPlanner::hasPlanWithVF(ElementCount VF, bool TF) const {
+ return any_of(VPlans, [VF, TF](const VPlanPtr &Plan) {
+ return Plan->hasVF(VF) && Plan->isCompatibleWithTF(TF);
+ });
+}
+
static void addRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b2d179d8e6d98..ea4d4614dd80d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5145,6 +5145,15 @@ class VPlan {
is_contained(ScalarPH->getPredecessors(), getMiddleBlock());
}
+ /// Returns true if this VPlan's tail-folding matches \p TF.
+ /// A plan is tail-folded if it has a header mask, or if it has no scalar
+ /// tail while the TC is not evenly divisible by VF (in which case the tail
+ /// must be folded by the vector loop itself).
+ ///
+ /// This function is used to disambiguate VPlan lookup by VF when multiple
+ /// plans share the same VF but differ in tail-folding status.
+ bool isCompatibleWithTF(bool TF);
+
/// The type of the canonical induction variable of the vector loop.
Type *getIndexType() const { return VF.getType(); }
};
More information about the llvm-branch-commits
mailing list