[llvm] 3e4f56b - [VPlan] Introduce VPlan-based hasTailFolded helper. (NFC) (#208329)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 12:57:40 PDT 2026
Author: Florian Hahn
Date: 2026-07-09T19:57:34Z
New Revision: 3e4f56bdecac68443e37df22965ce47d6b0bd113
URL: https://github.com/llvm/llvm-project/commit/3e4f56bdecac68443e37df22965ce47d6b0bd113
DIFF: https://github.com/llvm/llvm-project/commit/3e4f56bdecac68443e37df22965ce47d6b0bd113.diff
LOG: [VPlan] Introduce VPlan-based hasTailFolded helper. (NFC) (#208329)
After having dedicated support for modeling the header mask of a region
created for tail-folding, add a new helper to query if a plan has it's
tail folded, i.e. it has a header mask (materialized or not).
Similarly to https://github.com/llvm/llvm-project/pull/207784, it also
adds a wrapper to LoopVectorizationPlanner that asserts that cost model
and VPlan decision agree. The wrapper should be removed after no
divergence are found.
PR: https://github.com/llvm/llvm-project/pull/208329
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index c5524d01fa76e..1e48aea0fc061 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -970,6 +970,10 @@ class LoopVectorizationPlanner {
/// loop. Asserts that the VPlan decision matches the legacy cost model.
bool requiresScalarEpilogue(VPlan &Plan, ElementCount VF) const;
+ /// Returns true if \p Plan folds the tail by masking. Asserts that the
+ /// VPlan-based decision matches the legacy cost model.
+ bool hasTailFolded(const VPlan &Plan) const;
+
/// Attach the runtime checks of \p RTChecks to \p Plan.
void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks,
bool HasBranchWeights) const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 19b77e40a4103..6e94d36cd8fb1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3620,7 +3620,7 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
}
if (Result.Width.isScalar() ||
- isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
+ isMoreProfitable(NextVF, Result, MaxTripCount, !hasTailFolded(MainPlan),
/*IsEpilogue*/ true)) {
Result = NextVF;
BestPlan = &CurrentPlan;
@@ -5672,7 +5672,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
// TODO: Remove this code after stepping away from the legacy cost model and
// adding code to simplify VPlans before calculating their costs.
auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- if (TC == VF && !CM.foldTailByMasking())
+ if (TC == VF && !hasTailFolded(Plan))
addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
CostCtx.SkipCostComputation);
@@ -5929,7 +5929,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
}
if (CM.maskPartialAliasing()) {
- assert(CM.foldTailByMasking() && "Expected tail folding to be enabled");
+ assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
*CM.Legal->getRuntimePointerChecking()->getDiffChecks(),
HasBranchWeights);
@@ -5969,6 +5969,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
// Regions are dissolved after optimizing for VF and UF, which completely
// removes unneeded loop regions first.
+ const bool HasTailFolded = hasTailFolded(BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
// Expand BranchOnTwoConds after dissolution, when latch has direct access to
// its successors.
@@ -5986,7 +5987,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
assert((OrigLoop->getUniqueLatchExitBlock() || RequiresScalarEpilogue) &&
"loops not exiting via the latch without required epilogue?");
VPlanTransforms::materializeVectorTripCount(
- BestVPlan, VectorPH, CM.foldTailByMasking(), RequiresScalarEpilogue,
+ BestVPlan, VectorPH, HasTailFolded, RequiresScalarEpilogue,
&BestVPlan.getVFxUF(), MaxRuntimeStep);
VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
// Limit expansions to VPInstruction to when not vectorizing the epilogue.
@@ -6197,9 +6198,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
: VPI->getOperand(1);
if (Consecutive) {
Builder.setInsertPoint(VPI);
- Ptr = Builder.createConsecutiveVectorPointer(
- Ptr, getLoadStoreType(I), Reverse, CM.foldTailByMasking(),
- VPI->getDebugLoc());
+ Ptr = Builder.createConsecutiveVectorPointer(Ptr, getLoadStoreType(I),
+ Reverse, Plan.hasTailFolded(),
+ VPI->getDebugLoc());
}
if (Reverse && Mask)
@@ -6638,8 +6639,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
*Plan, *TLI))
return nullptr;
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- /*FoldTail=*/false);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
return Plan;
}
@@ -6808,8 +6808,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Optimize FindIV reductions to use sentinel-based approach when possible.
RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
*OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- CM.foldTailByMasking());
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
@@ -7126,6 +7125,13 @@ bool LoopVectorizationPlanner::requiresScalarEpilogue(VPlan &Plan,
return Result;
}
+bool LoopVectorizationPlanner::hasTailFolded(const VPlan &Plan) const {
+ bool Result = Plan.hasTailFolded();
+ assert(CM.foldTailByMasking() == Result &&
+ "CM.foldTailByMasking and the VPlan-based check must agree");
+ return Result;
+}
+
void LoopVectorizationPlanner::addMinimumIterationCheck(
VPlan &Plan, ElementCount VF, unsigned UF,
ElementCount MinProfitableTripCount) const {
@@ -7135,7 +7141,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
: nullptr;
RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
MinProfitableTripCount, requiresScalarEpilogue(Plan, VF),
- CM.foldTailByMasking(), OrigLoop, BranchWeights,
+ hasTailFolded(Plan), OrigLoop, BranchWeights,
OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
PSE, Plan.getEntry());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3f9bb6f44497c..eefe02b3e41ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4894,6 +4894,12 @@ class VPlan {
/// loop region contains a nested loop region.
LLVM_ABI_FOR_TEST bool isOuterLoop() const;
+ /// Returns true if the vector loop region is tail-folded.
+ bool hasTailFolded() const {
+ const VPRegionBlock *LoopRegion = getVectorLoopRegion();
+ return LoopRegion && LoopRegion->getHeaderMask();
+ }
+
/// Returns the 'middle' block of the plan, that is the block that selects
/// whether to execute the scalar tail loop or the exit block from the loop
/// latch. If there is an early exit from the vector loop, the middle block
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1bae8b6402932..343ee02007966 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1153,14 +1153,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
}
void VPlanTransforms::optimizeInductionLiveOutUsers(
- VPlan &Plan, PredicatedScalarEvolution &PSE, bool FoldTail) {
+ VPlan &Plan, PredicatedScalarEvolution &PSE) {
// Compute end values for all inductions.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
VPBuilder VectorPHBuilder(VectorPH, VectorPH->begin());
DenseMap<VPValue *, VPValue *> EndValues;
VPValue *ResumeTC =
- FoldTail ? Plan.getTripCount() : &Plan.getVectorTripCount();
+ Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
auto *WideIV = dyn_cast<VPWidenInductionRecipe>(&Phi);
if (!WideIV)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 2158d6a0cc67d..85375625d34b5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -416,8 +416,7 @@ struct VPlanTransforms {
/// IV values by feeding them precomputed end values instead, possibly taken
/// one step backwards.
static void optimizeInductionLiveOutUsers(VPlan &Plan,
- PredicatedScalarEvolution &PSE,
- bool FoldTail);
+ PredicatedScalarEvolution &PSE);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
More information about the llvm-commits
mailing list