[llvm] [VPlan] Model epilogue skeleton in VPlan by wrapping in VPIRBBs (NFCI). (PR #222312)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 08:53:05 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/222312
>From 273ec8dca45c8a5d74a5c09b4f039d2246781169 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 23 Aug 2026 13:35:41 +0100
Subject: [PATCH 1/3] [VPlan] Model epilogue skeleton in VPlan by wrapping in
VPIRBBs (NFCI).
Replace most of connectEpilogueVectorLoop by modeling the CFG explicitly
in the epilogue VPlan, by importing the CFG for the skeleton created after
executing the main loop.
To do so, update the main loop's plan to replace its VPBasicBlocks with
VPIRBasicBlocks wrapping the generated IR blocks. This allows accessing
the CFG for the VPlan, after execute.
A new modelGeneratedMainLoopBlocks transform then duplicates the CFG
from the main plan in the epilogue plan. This includes all runtime
checks and bypass blocks. This means we no longer need a number of
manual patching up of the IR and DT to connect main and epilogue plans
after execute.
It is another step getting us slightly closer to completely modeling
epilogue vectorization natively in VPlan.
Along the way, it will also help to simplify code needed to perform
folds on the epilogue skeleton
(i.e. simplify https://github.com/llvm/llvm-project/pull/204243).
---
.../Transforms/Vectorize/LoopVectorize.cpp | 138 ++++++----------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 31 ++--
.../Vectorize/VPlanConstruction.cpp | 52 ++++++
.../Transforms/Vectorize/VPlanTransforms.h | 8 +
.../VPlan/epilogue-vectorization-printing.ll | 152 +++++++++++++++---
5 files changed, 263 insertions(+), 118 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2493d2aca6265..4e7a25b8887ac 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -622,8 +622,6 @@ struct EpilogueLoopVectorizationInfo {
unsigned MainLoopUF = 0;
ElementCount EpilogueVF = ElementCount::getFixed(0);
unsigned EpilogueUF = 0;
- BasicBlock *MainLoopIterationCountCheck = nullptr;
- BasicBlock *EpilogueIterationCountCheck = nullptr;
Value *VectorTripCount = nullptr;
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
@@ -687,16 +685,22 @@ class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
// vectorization of *epilogue* loops in the process of vectorizing loops and
// their epilogues.
class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
+ VPlan &MainPlan;
+
public:
+ BasicBlock *VecEpilogueIterationCountCheck = nullptr;
+
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
LoopInfo *LI, DominatorTree *DT,
const TargetTransformInfo *TTI,
AssumptionCache *AC,
EpilogueLoopVectorizationInfo &EPI,
- GeneratedRTChecks &Checks, VPlan &Plan)
+ GeneratedRTChecks &Checks, VPlan &Plan,
+ VPlan &MainPlan)
: InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI,
Checks, Plan, EPI.EpilogueVF,
- EPI.EpilogueUF) {}
+ EPI.EpilogueUF),
+ MainPlan(MainPlan) {}
/// Implements the interface for creating a vectorized skeleton using the
/// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
BasicBlock *createVectorizedLoopSkeleton() final;
@@ -1961,7 +1965,7 @@ static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
VPlan *Plan = nullptr) {
if (!Plan)
Plan = VPBB->getPlan();
- VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
+ VPIRBasicBlock *IRVPBB = Plan->createEmptyVPIRBasicBlock(IRBB);
auto IP = IRVPBB->begin();
for (auto &R : make_early_inc_range(VPBB->phis()))
R.moveBefore(*IRVPBB, IP);
@@ -5873,6 +5877,14 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ILV.printDebugTracesAtEnd();
+ // Wrap the generated blocks in VPIRBasicBlocks, so they can be used in the
+ // epilogue plan.
+ if (EpilogueVecKind == EpilogueVectorizationKind::MainLoop)
+ for (VPBasicBlock *VPBB : to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(
+ vp_depth_first_shallow(BestVPlan.getEntry()))))
+ if (!isa<VPIRBasicBlock>(VPBB))
+ replaceVPBBWithIRVPBB(VPBB, State.CFG.VPBB2IRBB.at(VPBB), &BestVPlan);
+
return ExpandedSCEVs;
}
@@ -5919,8 +5931,12 @@ BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
}
VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
- Plan.setEntry(NewEntry);
- // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
+
+ VecEpilogueIterationCountCheck = OriginalScalarPH;
+
+ // Model the skeleton from the main vector loop in the epilogue plan.
+ RUN_VPLAN_PASS(VPlanTransforms::modelGeneratedMainLoopBlocks, Plan, MainPlan,
+ NewEntry);
return OriginalScalarPH;
}
@@ -7619,19 +7635,10 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
}
static void
-fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
- VPlan &BestEpiPlan,
+fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, VPlan &BestEpiPlan,
ArrayRef<VPInstruction *> ResumeValues) {
- // Fix resume values from the additional bypass block.
- BasicBlock *PH = L->getLoopPreheader();
- for (auto *Pred : predecessors(PH)) {
- for (PHINode &Phi : PH->phis()) {
- if (Phi.getBasicBlockIndex(Pred) != -1)
- continue;
- Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
- }
- }
auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
+ BasicBlock *PH = ScalarPH->getIRBasicBlock();
if (ScalarPH->hasPredecessors()) {
// Fix resume values for inductions and reductions from the additional
// bypass block using the incoming values from the main loop's resume phis.
@@ -7651,51 +7658,33 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
}
/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
-/// loop, after both plans have executed, updating branches from the iteration
-/// and runtime checks of the main loop, as well as updating various phis. \p
+/// loop, after both plans have executed, updating the branch from the iteration
+/// count check of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
- EpilogueLoopVectorizationInfo &EPI,
- DominatorTree *DT,
- GeneratedRTChecks &Checks,
- ArrayRef<Instruction *> InstsToMove,
- ArrayRef<VPInstruction *> ResumeValues) {
- BasicBlock *VecEpilogueIterationCountCheck =
- cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
-
+static void
+connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
+ BasicBlock *VecEpilogueIterationCountCheck,
+ ArrayRef<Instruction *> InstsToMove,
+ ArrayRef<VPInstruction *> ResumeValues) {
+ // The main loop's iteration count check is the predecessor bypassing the main
+ // vector loop, i.e. branching here as its first successor.
+ BasicBlock *MainLoopIterationCountCheck = *find_if(
+ predecessors(VecEpilogueIterationCountCheck), [&](BasicBlock *P) {
+ auto *BI = dyn_cast<CondBrInst>(P->getTerminator());
+ return BI && BI->getSuccessor(0) == VecEpilogueIterationCountCheck;
+ });
BasicBlock *VecEpiloguePreHeader =
cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator())
->getSuccessor(1);
- // Adjust the control flow taking the state info from the main loop
- // vectorization into account.
- assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
- "expected this to be saved from the previous pass.");
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
- // Helper to redirect an edge from \p BB to \p VecEpilogueIterationCountCheck
- // to \p NewSucc instead, updating the DomTree.
- auto RedirectEdge = [&](BasicBlock *BB, BasicBlock *NewSucc) {
- BB->getTerminator()->replaceUsesOfWith(VecEpilogueIterationCountCheck,
- NewSucc);
- DTU.applyUpdates(
- {{DominatorTree::Delete, BB, VecEpilogueIterationCountCheck},
- {DominatorTree::Insert, BB, NewSucc}});
- };
-
- RedirectEdge(EPI.MainLoopIterationCountCheck, VecEpiloguePreHeader);
-
- BasicBlock *ScalarPH =
- cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
- RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
-
- // Adjust the terminators of runtime check blocks and phis using them.
- BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
- BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
- if (SCEVCheckBlock)
- RedirectEdge(SCEVCheckBlock, ScalarPH);
- if (MemCheckBlock)
- RedirectEdge(MemCheckBlock, ScalarPH);
+ MainLoopIterationCountCheck->getTerminator()->replaceSuccessorWith(
+ VecEpilogueIterationCountCheck, VecEpiloguePreHeader);
+ DTU.applyUpdates({{DominatorTree::Delete, MainLoopIterationCountCheck,
+ VecEpilogueIterationCountCheck},
+ {DominatorTree::Insert, MainLoopIterationCountCheck,
+ VecEpiloguePreHeader}});
// The vec.epilog.iter.check block may contain Phi nodes from inductions
// or reductions which merge control-flow from the latch block and the
@@ -7709,20 +7698,6 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi->replaceIncomingBlockWith(
VecEpilogueIterationCountCheck->getSinglePredecessor(),
VecEpilogueIterationCountCheck);
-
- // If the phi doesn't have an incoming value from the
- // EpilogueIterationCountCheck, we are done. Otherwise remove the
- // incoming value and also those from other check blocks. This is needed
- // for reduction phis only.
- if (none_of(Phi->blocks(), [&](BasicBlock *IncB) {
- return EPI.EpilogueIterationCountCheck == IncB;
- }))
- continue;
- for (BasicBlock *BB :
- {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
- if (BB)
- Phi->removeIncomingValue(BB);
- }
}
auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
@@ -7732,7 +7707,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
// VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
// after executing the main loop. We need to update the resume values of
// inductions and reductions during epilogue vectorization.
- fixScalarResumeValuesFromBypass(VecEpilogueIterationCountCheck, L, EpiPlan,
+ fixScalarResumeValuesFromBypass(VecEpilogueIterationCountCheck, EpiPlan,
ResumeValues);
// Remove dead phis that were moved to the epilogue preheader but are unused
@@ -8202,28 +8177,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
++LoopsVectorized;
- // Derive EPI fields from VPlan-generated IR.
BasicBlock *EntryBB =
cast<VPIRBasicBlock>(BestMainPlan.getEntry())->getIRBasicBlock();
EntryBB->setName("iter.check");
- EPI.EpilogueIterationCountCheck = EntryBB;
- // The check chain is: Entry -> [SCEV] -> [Mem] -> MainCheck -> VecPH.
- // MainCheck is the non-bypass successor of the last runtime check block
- // (or Entry if there are no runtime checks).
- BasicBlock *LastCheck = EntryBB;
- if (BasicBlock *MemBB = Checks.getMemRuntimeChecks().second)
- LastCheck = MemBB;
- else if (BasicBlock *SCEVBB = Checks.getSCEVChecks().second)
- LastCheck = SCEVBB;
- BasicBlock *ScalarPH = L->getLoopPreheader();
- auto *BI = cast<CondBrInst>(LastCheck->getTerminator());
- EPI.MainLoopIterationCountCheck =
- BI->getSuccessor(BI->getSuccessor(0) == ScalarPH);
// Second pass vectorizes the epilogue and adjusts the control flow
// edges from the first pass.
EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI,
- Checks, BestEpiPlan);
+ Checks, BestEpiPlan, BestMainPlan);
SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
BestMainPlan, BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
*PSE.getSE(), ResumeValues);
@@ -8231,8 +8192,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LVP.executePlan(
EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
- connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
- ResumeValues);
+ connectEpilogueVectorLoop(BestEpiPlan, DT,
+ EpilogILV.VecEpilogueIterationCountCheck,
+ InstsToMove, ResumeValues);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8b42ccb6803c8..bf3d786a68e50 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -476,19 +476,28 @@ void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
} else {
// Set each forward successor here when it is created, excluding
// backedges. A backward successor is set when the branch is created.
- // Branches to VPIRBasicBlocks must have the same successors in VPlan as
- // in the original IR, except when the predecessor is the entry block.
- // This enables including SCEV and memory runtime check blocks in VPlan.
- // TODO: Remove exception by modeling the terminator of entry block using
+ // Generated successors are redirected, as for the entry block and for
+ // blocks bypassing both vector loops during epilogue vectorization. Edges
+ // already present in the generated IR need no update; this happens during
+ // epilogue vectorization, where the plan models blocks generated for the
+ // main vector loop.
+ // TODO: Remove the exception by modeling those terminators using
// BranchOnCond.
- unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
auto *TermBr = cast<CondBrInst>(PredBBTerminator);
- assert((!TermBr->getSuccessor(idx) ||
- (isa<VPIRBasicBlock>(this) &&
- (TermBr->getSuccessor(idx) == NewBB ||
- PredVPBlock == getPlan()->getEntry()))) &&
- "Trying to reset an existing successor block.");
- TermBr->setSuccessor(idx, NewBB);
+ if (TermBr->getSuccessor(0) != NewBB &&
+ TermBr->getSuccessor(1) != NewBB) {
+ unsigned Idx = PredVPSuccessors.front() == this ? 0 : 1;
+ BasicBlock *ReplacedSucc = TermBr->getSuccessor(Idx);
+ assert(
+ (!ReplacedSucc || isa<VPIRBasicBlock>(PredVPBB)) &&
+ "only VPIRBasicBlock predecessors may have an existing successor "
+ "redirected");
+ if (ReplacedSucc)
+ ReplacedSucc->removePredecessor(PredBB, /*KeepOneInputPHIs=*/true);
+ TermBr->setSuccessor(Idx, NewBB);
+ if (ReplacedSucc)
+ CFG.DTU.applyUpdates({{DominatorTree::Delete, PredBB, ReplacedSucc}});
+ }
}
CFG.DTU.applyUpdates({{DominatorTree::Insert, PredBB, NewBB}});
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c0c63f3e87842..e9937e32a6e79 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1429,6 +1429,58 @@ static void insertCheckBlockBeforeVectorLoop(VPlan &Plan,
addIncomingForLastPredecessor(ScalarPH);
}
+void VPlanTransforms::modelGeneratedMainLoopBlocks(
+ VPlan &Plan, VPlan &MainPlan, VPIRBasicBlock *EnteredFrom) {
+ // Map blocks from MainPlan to new, empty VPIRBasicBlocks in Plan, so the
+ // skeleton CFG can be modeled explicitly. MainPlan's entry maps to Plan's
+ // now-disconnected entry, its scalar PH to EnteredFrom, mapped last so it can
+ // be dropped as source below. Blocks without successors have no edges.
+ auto *MainEntry = cast<VPIRBasicBlock>(MainPlan.getEntry());
+ auto *MainScalarPH = cast<VPIRBasicBlock>(MainPlan.getScalarPreheader());
+ SmallVector<VPIRBasicBlock *> BypassBlocks;
+ SmallMapVector<VPIRBasicBlock *, VPIRBasicBlock *, 8> Old2NewVPBB;
+ Old2NewVPBB[MainEntry] = cast<VPIRBasicBlock>(Plan.getEntry());
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ MainEntry);
+ for (VPIRBasicBlock *VPBB : VPBlockUtils::blocksAs<VPIRBasicBlock>(RPOT)) {
+ // Collect bypass blocks (minimum iteration checks, runtime checks).
+ if (VPBB->getNumSuccessors() == 2 &&
+ VPBB->getSuccessors()[0] == MainScalarPH)
+ BypassBlocks.push_back(VPBB);
+ if (VPBB != MainEntry && VPBB != MainScalarPH && VPBB->hasSuccessors())
+ Old2NewVPBB[VPBB] =
+ Plan.createEmptyVPIRBasicBlock(VPBB->getIRBasicBlock());
+ }
+ Old2NewVPBB[MainScalarPH] = EnteredFrom;
+
+ // First, connect the edges from the bypass blocks to the scalar preheader, in
+ // reverse order, to preserve the predecessor order of the generated IR. The
+ // last bypass block branches into Plan and is mirrored below.
+ VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
+ for (VPIRBasicBlock *MainVPBB : reverse(drop_end(BypassBlocks))) {
+ VPBlockUtils::connectBlocks(Old2NewVPBB[MainVPBB], ScalarPH);
+ addIncomingForLastPredecessor(ScalarPH);
+ }
+
+ // Mirror MainPlan's CFG, skipping the bypass edges connected above, which
+ // come first, and edges to blocks not modeled in Plan.
+ for (auto &[MainVPBB, VPBB] : drop_end(Old2NewVPBB))
+ for (VPBlockBase *Succ :
+ drop_begin(MainVPBB->getSuccessors(), VPBB->getNumSuccessors()))
+ if (auto *SuccVPBB = Old2NewVPBB.lookup(cast<VPIRBasicBlock>(Succ)))
+ VPBlockUtils::connectBlocks(VPBB, SuccVPBB);
+
+ // EnteredFrom is the only modeled block with phis; re-use the incoming values
+ // its IR phis already have for the new predecessors.
+ for (VPRecipeBase &R : EnteredFrom->phis()) {
+ auto *PhiR = cast<VPIRPhi>(&R);
+ for (VPIRBasicBlock *Pred :
+ VPBlockUtils::blocksAs<VPIRBasicBlock>(EnteredFrom->getPredecessors()))
+ PhiR->addIncoming(Plan.getOrAddLiveIn(
+ PhiR->getIRPhi().getIncomingValueForBlock(Pred->getIRBasicBlock())));
+ }
+}
+
// Likelyhood of bypassing the vectorized loop due to a runtime check block,
// including memory overlap checks block and wrapping/unit-stride checks block.
static constexpr uint32_t CheckBypassWeights[] = {1, 127};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6686d887bcf83..90f28b1fb4780 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -230,6 +230,14 @@ struct VPlanTransforms {
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock,
bool AddBranchWeights);
+ /// Model the blocks the executed \p MainPlan generated for the main vector
+ /// loop in \p Plan during epilogue vectorization, wrapping each in a
+ /// VPIRBasicBlock, with \p EnteredFrom the block \p Plan is entered from.
+ /// Edges from blocks bypassing both vector loops are redirected to \p Plan's
+ /// scalar preheader, all others are mirrored.
+ static void modelGeneratedMainLoopBlocks(VPlan &Plan, VPlan &MainPlan,
+ VPIRBasicBlock *EnteredFrom);
+
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes. Returns false if any VPInstructions could not be converted
/// to a wide recipe if needed. Uses \p PSE to detect contiguous memory
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
index 9bf623bfaef49..fd4b2af5448d3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
@@ -69,9 +69,25 @@ define i64 @resume_values(ptr noalias %A, i64 %n) {
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %min.iters.check = icmp ult i64 %n, 4
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
+; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ]
-; CHECK-NEXT: IR %bc.merge.rdx = phi i64 [ %4, %middle.block ], [ 5, %iter.check ], [ 5, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<%n.vec> from ir-bb<middle.block>)
+; CHECK-NEXT: IR %bc.merge.rdx = phi i64 [ %4, %middle.block ], [ 5, %iter.check ], [ 5, %vector.main.loop.iter.check ] (extra operands: ir<5> from ir-bb<vector.main.loop.iter.check>, ir<%4> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<%0>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -104,8 +120,8 @@ define i64 @resume_values(ptr noalias %A, i64 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, vec.epilog.middle.block ], [ ir<5>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, vec.epilog.middle.block ], [ ir<5>, ir-bb<vec.epilog.iter.check> ], [ ir<5>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -224,9 +240,33 @@ define i64 @bypass_blocks(ptr %A, ptr %B, i32 %n) {
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %A2 = ptrtoaddr ptr %A to i64
+; CHECK-NEXT: IR %B1 = ptrtoaddr ptr %B to i64
+; CHECK-NEXT: IR %min.iters.check = icmp ult i32 %n, 4
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.scevcheck>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.scevcheck>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.memcheck>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.memcheck>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
+; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ]
-; CHECK-NEXT: IR %bc.merge.rdx = phi i64 [ %10, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<%n.vec> from ir-bb<middle.block>)
+; CHECK-NEXT: IR %bc.merge.rdx = phi i64 [ %10, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<%10> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<%4>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -262,8 +302,8 @@ define i64 @bypass_blocks(ptr %A, ptr %B, i32 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<vector.memcheck> ], [ ir<0>, ir-bb<vector.scevcheck> ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<vector.memcheck> ], [ ir<0>, ir-bb<vector.scevcheck> ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -369,9 +409,30 @@ define void @all_iterations_in_main_loop_with_memcheck(ptr %dst, ptr %src) {
; CHECK-LABEL: VPlan for loop in 'all_iterations_in_main_loop_with_memcheck'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<16> = vector-trip-count
+; CHECK-NEXT: Live-in ir<16> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %src2 = ptrtoaddr ptr %src to i64
+; CHECK-NEXT: IR %dst1 = ptrtoaddr ptr %dst to i64
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.memcheck>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.memcheck>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ 16, %middle.block ], [ 0, %iter.check ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ 16, %middle.block ], [ 0, %iter.check ], [ 0, %vector.memcheck ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<16> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<0>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -399,7 +460,7 @@ define void @all_iterations_in_main_loop_with_memcheck(ptr %dst, ptr %src) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<16>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<16>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<vector.memcheck> ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -499,10 +560,29 @@ define void @all_iterations_in_main_loop_with_scevcheck(ptr %p, i32 %off) {
; CHECK-LABEL: VPlan for loop in 'all_iterations_in_main_loop_with_scevcheck'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<16> = vector-trip-count
+; CHECK-NEXT: Live-in ir<16> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.scevcheck>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.scevcheck>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ 16, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.main.loop.iter.check ]
-; CHECK-NEXT: IR %bc.resume.val = phi i32 [ %2, %middle.block ], [ %off, %iter.check ], [ %off, %vector.scevcheck ], [ %off, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ 16, %middle.block ], [ 0, %iter.check ], [ 0, %vector.scevcheck ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<16> from ir-bb<middle.block>)
+; CHECK-NEXT: IR %bc.resume.val = phi i32 [ %2, %middle.block ], [ %off, %iter.check ], [ %off, %vector.scevcheck ], [ %off, %vector.main.loop.iter.check ] (extra operands: ir<%off> from ir-bb<vector.main.loop.iter.check>, ir<%2> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<0>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -531,8 +611,8 @@ define void @all_iterations_in_main_loop_with_scevcheck(ptr %p, i32 %off) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<16>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP2]]>, vec.epilog.middle.block ], [ ir<%off>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<16>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<vector.scevcheck> ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP2]]>, vec.epilog.middle.block ], [ ir<%off>, ir-bb<vec.epilog.iter.check> ], [ ir<%off>, ir-bb<vector.scevcheck> ], [ ir<%off>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -618,8 +698,25 @@ define void @dead_main_vector_loop(ptr %dst, i64 %n) {
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<%clamped> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %clamped = call i64 @llvm.umin.i64(i64 %n, i64 4)
+; CHECK-NEXT: IR %min.iters.check = icmp ult i64 %clamped, 4
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
+; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<%n.vec> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<%clamped>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -643,7 +740,7 @@ define void @dead_main_vector_loop(ptr %dst, i64 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, vec.epilog.middle.block ], [ ir<0>, ir-bb<vec.epilog.iter.check> ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -737,9 +834,26 @@ define i32 @nested_loop(ptr noalias %p, ptr noalias %end, ptr noalias %dst, i64
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<%3> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer.header>:
+; CHECK-NEXT: IR %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+; CHECK-NEXT: IR %min.iters.check = icmp ult i64 %3, 4
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, ir-bb<vector.main.loop.iter.check>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.main.loop.iter.check>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>, ir-bb<vector.ph>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: Successor(s): ir-bb<vector.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<vector.body>:
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.iter.check>
+; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.iter.check>:
-; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ]
-; CHECK-NEXT: IR %bc.resume.val = phi ptr [ %6, %middle.block ], [ %p, %iter.check ], [ %p, %vector.main.loop.iter.check ]
+; CHECK-NEXT: IR %vec.epilog.resume.val = phi i64 [ %n.vec, %middle.block ], [ 0, %iter.check ], [ 0, %vector.main.loop.iter.check ] (extra operands: ir<0> from ir-bb<vector.main.loop.iter.check>, ir<%n.vec> from ir-bb<middle.block>)
+; CHECK-NEXT: IR %bc.resume.val = phi ptr [ %6, %middle.block ], [ %p, %iter.check ], [ %p, %vector.main.loop.iter.check ] (extra operands: ir<%p> from ir-bb<vector.main.loop.iter.check>, ir<%6> from ir-bb<middle.block>)
; CHECK-NEXT: EMIT vp<%min.epilog.iters.check> = icmp ult ir<%4>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.epilog.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<vec.epilog.scalar.ph>, vec.epilog.ph
@@ -773,7 +887,7 @@ define i32 @nested_loop(ptr noalias %p, ptr noalias %end, ptr noalias %dst, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vec.epilog.scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, vec.epilog.middle.block ], [ ir<%p>, ir-bb<vec.epilog.iter.check> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, vec.epilog.middle.block ], [ ir<%p>, ir-bb<vec.epilog.iter.check> ], [ ir<%p>, ir-bb<outer.header> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
>From f3d96f6404d81def940b6db623f77e7e679455f2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 10 Sep 2026 15:44:11 +0100
Subject: [PATCH 2/3] !fixup address comments, thanks
---
.../Vectorize/VPlanConstruction.cpp | 50 +++++++++----------
.../Transforms/Vectorize/VPlanTransforms.h | 10 ++--
2 files changed, 28 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1eaf104bba111..2f498001ef92b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1462,44 +1462,40 @@ static void insertCheckBlockBeforeVectorLoop(VPlan &Plan,
}
void VPlanTransforms::modelGeneratedMainLoopBlocks(
- VPlan &Plan, VPlan &MainPlan, VPIRBasicBlock *EnteredFrom) {
- // Map blocks from MainPlan to new, empty VPIRBasicBlocks in Plan, so the
- // skeleton CFG can be modeled explicitly. MainPlan's entry maps to Plan's
- // now-disconnected entry, its scalar PH to EnteredFrom, mapped last so it can
- // be dropped as source below. Blocks without successors have no edges.
- auto *MainEntry = cast<VPIRBasicBlock>(MainPlan.getEntry());
- auto *MainScalarPH = cast<VPIRBasicBlock>(MainPlan.getScalarPreheader());
- SmallVector<VPIRBasicBlock *> BypassBlocks;
- SmallMapVector<VPIRBasicBlock *, VPIRBasicBlock *, 8> Old2NewVPBB;
- Old2NewVPBB[MainEntry] = cast<VPIRBasicBlock>(Plan.getEntry());
+ VPlan &EpiPlan, VPlan &MainPlan, VPIRBasicBlock *EnteredFrom) {
+ // Map blocks from MainPlan to new, empty VPIRBasicBlocks in EpiPlan, so the
+ // skeleton CFG can be modeled explicitly. MainPlan's entry maps to EpiPlan's
+ // now-disconnected entry and its scalar PH to EnteredFrom.
+ VPBlockBase *MainEntry = MainPlan.getEntry();
+ VPBlockBase *MainScalarPH = MainPlan.getScalarPreheader();
+ SmallMapVector<VPBlockBase *, VPBlockBase *, 8> Old2NewVPBB;
+ Old2NewVPBB[MainEntry] = EpiPlan.getEntry();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
MainEntry);
- for (VPIRBasicBlock *VPBB : VPBlockUtils::blocksAs<VPIRBasicBlock>(RPOT)) {
- // Collect bypass blocks (minimum iteration checks, runtime checks).
- if (VPBB->getNumSuccessors() == 2 &&
- VPBB->getSuccessors()[0] == MainScalarPH)
- BypassBlocks.push_back(VPBB);
+ for (VPIRBasicBlock *VPBB : VPBlockUtils::blocksAs<VPIRBasicBlock>(RPOT))
+ // Skip entry block and exit blocks/scalar loop header; they are already
+ // modeling in the epilogue plan.
if (VPBB != MainEntry && VPBB != MainScalarPH && VPBB->hasSuccessors())
Old2NewVPBB[VPBB] =
- Plan.createEmptyVPIRBasicBlock(VPBB->getIRBasicBlock());
- }
+ EpiPlan.createEmptyVPIRBasicBlock(VPBB->getIRBasicBlock());
Old2NewVPBB[MainScalarPH] = EnteredFrom;
- // First, connect the edges from the bypass blocks to the scalar preheader, in
- // reverse order, to preserve the predecessor order of the generated IR. The
- // last bypass block branches into Plan and is mirrored below.
- VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
- for (VPIRBasicBlock *MainVPBB : reverse(drop_end(BypassBlocks))) {
- VPBlockUtils::connectBlocks(Old2NewVPBB[MainVPBB], ScalarPH);
- addIncomingForLastPredecessor(ScalarPH);
+ // First, connect the edges from the bypass blocks (minimum iteration checks,
+ // runtime checks) to the scalar preheader, in reverse order, to preserve the
+ // predecessor order of the generated IR.
+ VPBasicBlock *EpiScalarPH = EpiPlan.getScalarPreheader();
+ for (VPBlockBase *MainVPBB :
+ reverse(drop_end(drop_begin(MainScalarPH->predecessors())))) {
+ VPBlockUtils::connectBlocks(Old2NewVPBB.lookup(MainVPBB), EpiScalarPH);
+ addIncomingForLastPredecessor(EpiScalarPH);
}
// Mirror MainPlan's CFG, skipping the bypass edges connected above, which
- // come first, and edges to blocks not modeled in Plan.
+ // come first, and edges to blocks not modeled in EpiPlan.
for (auto &[MainVPBB, VPBB] : drop_end(Old2NewVPBB))
for (VPBlockBase *Succ :
drop_begin(MainVPBB->getSuccessors(), VPBB->getNumSuccessors()))
- if (auto *SuccVPBB = Old2NewVPBB.lookup(cast<VPIRBasicBlock>(Succ)))
+ if (auto *SuccVPBB = Old2NewVPBB.lookup(Succ))
VPBlockUtils::connectBlocks(VPBB, SuccVPBB);
// EnteredFrom is the only modeled block with phis; re-use the incoming values
@@ -1508,7 +1504,7 @@ void VPlanTransforms::modelGeneratedMainLoopBlocks(
auto *PhiR = cast<VPIRPhi>(&R);
for (VPIRBasicBlock *Pred :
VPBlockUtils::blocksAs<VPIRBasicBlock>(EnteredFrom->getPredecessors()))
- PhiR->addIncoming(Plan.getOrAddLiveIn(
+ PhiR->addIncoming(EpiPlan.getOrAddLiveIn(
PhiR->getIRPhi().getIncomingValueForBlock(Pred->getIRBasicBlock())));
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 4d1351bdc3452..baacd7353d9f4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -233,11 +233,11 @@ struct VPlanTransforms {
bool AddBranchWeights);
/// Model the blocks the executed \p MainPlan generated for the main vector
- /// loop in \p Plan during epilogue vectorization, wrapping each in a
- /// VPIRBasicBlock, with \p EnteredFrom the block \p Plan is entered from.
- /// Edges from blocks bypassing both vector loops are redirected to \p Plan's
- /// scalar preheader, all others are mirrored.
- static void modelGeneratedMainLoopBlocks(VPlan &Plan, VPlan &MainPlan,
+ /// loop in \p EpiPlan during epilogue vectorization, wrapping each in a
+ /// VPIRBasicBlock, with \p EnteredFrom the block \p EpiPlan is entered from.
+ /// Edges from blocks bypassing both vector loops are redirected to \p
+ /// EpiPlan's scalar preheader, all others are mirrored.
+ static void modelGeneratedMainLoopBlocks(VPlan &EpiPlan, VPlan &MainPlan,
VPIRBasicBlock *EnteredFrom);
/// Replaces the VPInstructions in \p Plan with corresponding
>From 6c6c9bfbef3f4a4cef732bf076a944ef0c7894d7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 11 Sep 2026 15:13:44 +0100
Subject: [PATCH 3/3] !fixup apply comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 ++++++++-----------
.../Vectorize/VPlanConstruction.cpp | 20 +++++++--------
2 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fcd82b09d684b..231b05be08da9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -691,7 +691,7 @@ class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
VPlan &MainPlan;
public:
- BasicBlock *VecEpilogueIterationCountCheck = nullptr;
+ VPIRBasicBlock *VecEpilogueIterationCountCheck = nullptr;
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
LoopInfo *LI, DominatorTree *DT,
@@ -5994,7 +5994,7 @@ BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
- VecEpilogueIterationCountCheck = OriginalScalarPH;
+ VecEpilogueIterationCountCheck = NewEntry;
// Model the skeleton from the main vector loop in the epilogue plan.
RUN_VPLAN_PASS(VPlanTransforms::modelGeneratedMainLoopBlocks, Plan, MainPlan,
@@ -7652,18 +7652,15 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, VPlan &BestEpiPlan,
/// count check of the main loop, as well as updating various phis. \p
/// InstsToMove contains instructions that need to be moved to the preheader of
/// the epilogue vector loop.
-static void
-connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
- BasicBlock *VecEpilogueIterationCountCheck,
- ArrayRef<Instruction *> InstsToMove,
- ArrayRef<VPInstruction *> ResumeValues) {
- // The main loop's iteration count check is the predecessor bypassing the main
- // vector loop, i.e. branching here as its first successor.
- BasicBlock *MainLoopIterationCountCheck = *find_if(
- predecessors(VecEpilogueIterationCountCheck), [&](BasicBlock *P) {
- auto *BI = dyn_cast<CondBrInst>(P->getTerminator());
- return BI && BI->getSuccessor(0) == VecEpilogueIterationCountCheck;
- });
+static void connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
+ VPIRBasicBlock *VecEpilogueIterCheckVPBB,
+ ArrayRef<Instruction *> InstsToMove,
+ ArrayRef<VPInstruction *> ResumeValues) {
+ ArrayRef<VPBlockBase *> Preds = VecEpilogueIterCheckVPBB->getPredecessors();
+ BasicBlock *MainLoopIterationCountCheck =
+ cast<VPIRBasicBlock>(Preds.front())->getIRBasicBlock();
+ BasicBlock *VecEpilogueIterationCountCheck =
+ VecEpilogueIterCheckVPBB->getIRBasicBlock();
BasicBlock *VecEpiloguePreHeader =
cast<CondBrInst>(VecEpilogueIterationCountCheck->getTerminator())
->getSuccessor(1);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index f2f422ac1940e..b7ae73cd06713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1454,17 +1454,17 @@ void VPlanTransforms::modelGeneratedMainLoopBlocks(
// now-disconnected entry and its scalar PH to EnteredFrom.
VPBlockBase *MainEntry = MainPlan.getEntry();
VPBlockBase *MainScalarPH = MainPlan.getScalarPreheader();
- SmallMapVector<VPBlockBase *, VPBlockBase *, 8> Old2NewVPBB;
- Old2NewVPBB[MainEntry] = EpiPlan.getEntry();
+ SmallMapVector<VPBlockBase *, VPBlockBase *, 8> MainToEpiVPBB;
+ MainToEpiVPBB[MainEntry] = EpiPlan.getEntry();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
MainEntry);
for (VPIRBasicBlock *VPBB : VPBlockUtils::blocksAs<VPIRBasicBlock>(RPOT))
// Skip entry block and exit blocks/scalar loop header; they are already
- // modeling in the epilogue plan.
+ // modeled in the epilogue plan.
if (VPBB != MainEntry && VPBB != MainScalarPH && VPBB->hasSuccessors())
- Old2NewVPBB[VPBB] =
+ MainToEpiVPBB[VPBB] =
EpiPlan.createEmptyVPIRBasicBlock(VPBB->getIRBasicBlock());
- Old2NewVPBB[MainScalarPH] = EnteredFrom;
+ MainToEpiVPBB[MainScalarPH] = EnteredFrom;
// First, connect the edges from the bypass blocks (minimum iteration checks,
// runtime checks) to the scalar preheader, in reverse order, to preserve the
@@ -1472,17 +1472,17 @@ void VPlanTransforms::modelGeneratedMainLoopBlocks(
VPBasicBlock *EpiScalarPH = EpiPlan.getScalarPreheader();
for (VPBlockBase *MainVPBB :
reverse(drop_end(drop_begin(MainScalarPH->predecessors())))) {
- VPBlockUtils::connectBlocks(Old2NewVPBB.lookup(MainVPBB), EpiScalarPH);
+ VPBlockUtils::connectBlocks(MainToEpiVPBB.lookup(MainVPBB), EpiScalarPH);
addIncomingForLastPredecessor(EpiScalarPH);
}
// Mirror MainPlan's CFG, skipping the bypass edges connected above, which
// come first, and edges to blocks not modeled in EpiPlan.
- for (auto &[MainVPBB, VPBB] : drop_end(Old2NewVPBB))
+ for (auto &[MainVPBB, EpiVPBB] : drop_end(MainToEpiVPBB))
for (VPBlockBase *Succ :
- drop_begin(MainVPBB->getSuccessors(), VPBB->getNumSuccessors()))
- if (auto *SuccVPBB = Old2NewVPBB.lookup(Succ))
- VPBlockUtils::connectBlocks(VPBB, SuccVPBB);
+ drop_begin(MainVPBB->getSuccessors(), EpiVPBB->getNumSuccessors()))
+ if (auto *SuccVPBB = MainToEpiVPBB.lookup(Succ))
+ VPBlockUtils::connectBlocks(EpiVPBB, SuccVPBB);
// EnteredFrom is the only modeled block with phis; re-use the incoming values
// its IR phis already have for the new predecessors.
More information about the llvm-commits
mailing list