[llvm] [LV] Use a ResumeForEpilogue marker for the main vector TC (NFC) (PR #228942)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 14:50:24 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/228942
Previously the vector trip count of the main loop was recovered via IR based pattern matching.
Instead, add a ResumeForEpilogue marker for the main plan's vector trip count to its middle block, which records the generated value, and pass it to addMinimumVectorEpilogueIterationCheck as a VPValue. The check is only reached after the main vector loop, so it uses that value directly rather than the resume phi merging it with the bypass value.
This makes matching more robust and is another step towards modeling the full epilogue skeleton in VPlan.
>From 3d4cc51d9b64912cfa17344edc5c5feff1169ba4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 27 Sep 2026 11:09:24 +0100
Subject: [PATCH] [LV] Use a ResumeForEpilogue marker for the main vector trip
count (NFC).
Previously the vector trip count of the main loop was recovered via IR based
pattern matching.
Instead, add a ResumeForEpilogue marker for the main plan's vector trip count
to its middle block, which records the generated value, and pass it to
addMinimumVectorEpilogueIterationCheck as a VPValue. The check is only reached
after the main vector loop, so it uses that value directly rather than the
resume phi merging it with the bypass value.
This makes matching more robust and is another step towards modeling the
full epilogue skeleton in VPlan.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 65 ++++++++-----------
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +-
.../Vectorize/VPlanConstruction.cpp | 7 +-
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
.../VPlan/epilogue-vectorization-printing.ll | 6 ++
5 files changed, 40 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45fcdff01a6d0..3e025f9620e51 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -619,7 +619,6 @@ struct EpilogueLoopVectorizationInfo {
ElementCount MainLoopVF = ElementCount::getFixed(0);
unsigned MainLoopUF = 0;
ElementCount EpilogueVF = ElementCount::getFixed(0);
- Value *VectorTripCount = nullptr;
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
ElementCount EVF)
@@ -7277,10 +7276,17 @@ LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
!EnableLoopVectorization) {}
+/// ResumeForEpilogue markers in the main plan, used by the epilogue plan.
+struct MainPlanResumeMarkers {
+ VPInstruction *CanIVResume;
+ VPInstruction *VectorTC;
+ SmallVector<VPInstruction *> ResumeValues;
+};
+
/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
/// vectorization.
-static SmallVector<VPInstruction *>
-preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
+static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan,
+ VPlan &EpiPlan) {
using namespace VPlanPatternMatch;
// When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
// introduce multiple uses of undef/poison. If the reduction start value may
@@ -7315,6 +7321,12 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
match(Term, m_BranchOnCount(m_VPValue(), m_VPValue(VectorTC)));
assert(MatchedTC && "must match vector trip count");
+ VPBasicBlock *MiddleVPBB = MainPlan.getMiddleBlock();
+ VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+ VPInstruction *VectorTCMarker = MiddleBuilder.createNaryOp(
+ VPInstruction::ResumeForEpilogue,
+ {VectorTC, MainPlan.getZero(VectorTC->getScalarType())});
+
// If there is a suitable resume value for the canonical induction in the
// scalar (which will become vector) epilogue loop, use it and move it to the
// beginning of the scalar preheader. Otherwise create it below.
@@ -7342,14 +7354,14 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
// Create a ResumeForEpilogue for the canonical IV resume and its bypass value
// as the first non-phi, to keep them alive for the epilogue.
VPBuilder ResumeBuilder(MainScalarPH);
- ResumeBuilder.createNaryOp(VPInstruction::ResumeForEpilogue,
- {ResumePhi, ResumePhi->getOperand(1)});
+ VPInstruction *CanIVResume = ResumeBuilder.createNaryOp(
+ VPInstruction::ResumeForEpilogue, {ResumePhi, ResumePhi->getOperand(1)});
// Create ResumeForEpilogue instructions for the resume phis of the
// VPIRPhis and their bypass values in the scalar header of the main plan and
// return them so they can be used as resume values when vectorizing the
// epilogue.
- return to_vector(
+ auto ResumeValues = to_vector(
map_range(MainPlan.getScalarHeader()->phis(), [&](VPRecipeBase &R) {
assert(isa<VPIRPhi>(R) &&
"only VPIRPhis expected in the scalar header");
@@ -7358,6 +7370,7 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
return ResumeBuilder.createNaryOp(VPInstruction::ResumeForEpilogue,
{MainResumePhi, Bypass});
}));
+ return {CanIVResume, VectorTCMarker, std::move(ResumeValues)};
}
/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
@@ -7366,13 +7379,13 @@ static void preparePlanForEpilogueVectorLoop(
VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
EpilogueLoopVectorizationInfo &EPI, LoopVectorizationPlanner &LVP,
VFSelectionContext &Config, ScalarEvolution &SE,
- ArrayRef<VPInstruction *> ResumeValues) {
+ const MainPlanResumeMarkers &Markers) {
// Build a map from the scalar-header PHI to the ResumeForEpilogue markers
// from the main plan.
// TODO: Replace the IR PHI key.
DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
for (auto [HeaderPhi, ResumeForEpi] :
- zip_equal(MainPlan.getScalarHeader()->phis(), ResumeValues))
+ zip_equal(MainPlan.getScalarHeader()->phis(), Markers.ResumeValues))
IRPhiToResumeForEpi[&cast<VPIRPhi>(HeaderPhi).getIRPhi()] = ResumeForEpi;
VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
@@ -7383,30 +7396,7 @@ static void preparePlanForEpilogueVectorLoop(
// at the resume value from the main vector loop. Find the resume value
// created during execution of the main VPlan. Add this resume value as an
// offset to the canonical IV of the epilogue loop.
- using namespace llvm::PatternMatch;
- VPInstruction *ResumeForEpilogue =
- cast<VPInstruction>(&*MainPlan.getScalarPreheader()->getFirstNonPhi());
- Value *EPResumeVal = ResumeForEpilogue->getUnderlyingValue();
- if (auto *ResumePhi = dyn_cast<PHINode>(EPResumeVal)) {
- for (Value *Inc : ResumePhi->incoming_values()) {
- if (match(Inc, m_SpecificInt(0)))
- continue;
- assert(!EPI.VectorTripCount &&
- "Must only have a single non-zero incoming value");
- EPI.VectorTripCount = Inc;
- }
- // If we didn't find a non-zero vector trip count, all incoming values
- // must be zero, which also means the vector trip count is zero.
- if (!EPI.VectorTripCount) {
- assert(ResumePhi->getNumIncomingValues() > 0 &&
- all_of(ResumePhi->incoming_values(), match_fn(m_SpecificInt(0))) &&
- "all incoming values must be 0");
- EPI.VectorTripCount = ResumePhi->getIncomingValue(0);
- }
- } else {
- EPI.VectorTripCount = EPResumeVal;
- }
- VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
+ VPValue *VPV = Plan.getOrAddLiveIn(Markers.CanIVResume->getUnderlyingValue());
assert(all_of(IV->users(),
[](const VPUser *U) {
if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
@@ -7581,8 +7571,9 @@ static void preparePlanForEpilogueVectorLoop(
estimateElementCount(EPI.MainLoopVF * EPI.MainLoopUF, VScale);
unsigned EpilogueLoopStep = estimateElementCount(EPI.EpilogueVF, VScale);
RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
- EPI.VectorTripCount, Plan.requiresScalarEpilogue(),
- EPI.EpilogueVF, MainLoopStep, EpilogueLoopStep, SE);
+ Plan.getOrAddLiveIn(Markers.VectorTC->getUnderlyingValue()),
+ Plan.requiresScalarEpilogue(), EPI.EpilogueVF, MainLoopStep,
+ EpilogueLoopStep, SE);
}
static void
@@ -8100,7 +8091,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// factor) again shortly afterwards.
BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
- SmallVector<VPInstruction *> ResumeValues =
+ MainPlanResumeMarkers Markers =
preparePlanForMainVectorLoop(BestMainPlan, BestEpiPlan);
EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF);
@@ -8143,7 +8134,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Checks, BestEpiPlan, BestMainPlan);
preparePlanForEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L,
ExpandedSCEVs, EPI, LVP, Config,
- *PSE.getSE(), ResumeValues);
+ *PSE.getSE(), Markers);
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
LLVM_DEBUG({
dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
@@ -8158,7 +8149,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
});
connectEpilogueVectorLoop(BestEpiPlan, DT,
EpilogILV.VecEpilogueIterationCountCheck,
- ResumeValues);
+ Markers.ResumeValues);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index efdf3fa80bbe9..e628e54393cba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1398,8 +1398,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// The lane specifies an index into a vector formed by combining all vector
/// operands (all operands after the first one).
ExtractLane,
- /// Explicit user for the resume phi of the canonical induction in the main
- /// VPlan, used by the epilogue vector loop.
+ /// Explicit user for values in the main VPlan, used by the epilogue vector
+ /// loop.
ResumeForEpilogue,
/// Extracts the last active lane from a set of vectors. The first operand
/// is the default value if no lanes in the masks are active. Conceptually,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ad9fb0b648497..e647b6c1586b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1671,16 +1671,15 @@ void VPlanTransforms::addIterationCountCheckBlock(
}
void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
- VPlan &Plan, Value *VectorTripCount, bool RequiresScalarEpilogue,
+ VPlan &Plan, VPValue *MainVectorTripCount, bool RequiresScalarEpilogue,
ElementCount EpilogueVF, unsigned MainLoopStep, unsigned EpilogueLoopStep,
ScalarEvolution &SE) {
// Add the minimum iteration check for the epilogue vector loop.
VPValue *TC = Plan.getTripCount();
- Value *TripCount = TC->getLiveInIRValue();
VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
VPValue *VFxUF = Builder.createExpandSCEV(
- SE.getElementCount(TripCount->getType(), EpilogueVF, SCEV::FlagNUW));
- VPValue *Count = Builder.createSub(TC, Plan.getOrAddLiveIn(VectorTripCount),
+ SE.getElementCount(TC->getScalarType(), EpilogueVF, SCEV::FlagNUW));
+ VPValue *Count = Builder.createSub(TC, MainVectorTripCount,
DebugLoc::getUnknown(), "n.vec.remaining");
// Generate code to check if the loop's trip count is less than VF * UF of
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 30d584ab8f589..fdf17d07d70b5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -216,7 +216,7 @@ struct VPlanTransforms {
/// Add a check to \p Plan to see if the epilogue vector loop should be
/// executed.
static void addMinimumVectorEpilogueIterationCheck(
- VPlan &Plan, Value *VectorTripCount, bool RequiresScalarEpilogue,
+ VPlan &Plan, VPValue *MainVectorTripCount, bool RequiresScalarEpilogue,
ElementCount EpilogueVF, unsigned MainLoopStep, unsigned EpilogueLoopStep,
ScalarEvolution &SE);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
index 7e9dd7640efd6..58dc0a73418f6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
@@ -37,6 +37,7 @@ define i64 @resume_values(ptr noalias %A, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block, vector.body
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue vp<%n.vec>, ir<0>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = compute-reduction-result (add) ir<%red.next>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
@@ -205,6 +206,7 @@ define i64 @bypass_blocks(ptr %A, ptr %B, i32 %n) {
; CHECK-NEXT: Successor(s): middle.block, vector.body
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue vp<%n.vec>, ir<0>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add) ir<%red.next>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
@@ -382,6 +384,7 @@ define void @all_iterations_in_main_loop_with_memcheck(ptr %dst, ptr %src) {
; CHECK-NEXT: Successor(s): middle.block, vector.body
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue ir<16>, ir<0>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; CHECK-EMPTY:
@@ -531,6 +534,7 @@ define void @all_iterations_in_main_loop_with_scevcheck(ptr %p, i32 %off) {
; CHECK-NEXT: Successor(s): middle.block, vector.body
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue ir<16>, ir<0>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; CHECK-EMPTY:
@@ -672,6 +676,7 @@ define void @dead_main_vector_loop(ptr %dst, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue vp<%n.vec>, ir<0>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%clamped>, vp<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
@@ -806,6 +811,7 @@ define i32 @nested_loop(ptr noalias %p, ptr noalias %end, ptr noalias %dst, i64
; CHECK-NEXT: Successor(s): middle.block, vector.body
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<%{{.+}}> = resume-for-epilogue vp<%n.vec>, ir<0>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%3>, vp<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<inner.exit>, ir-bb<scalar.ph>
More information about the llvm-commits
mailing list