[llvm] [VPlan] Hook IR blocks into VPlan during skeleton creation (NFC) (PR #114292)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 25 13:25:42 PST 2024
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/114292
>From 1b89761c30d273ae665b6da4b2ede0675373f2b1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 11 Oct 2024 10:04:40 +0100
Subject: [PATCH 01/10] [VPlan] Hook IR blocks into VPlan during skeleton
creation (NFC)
As a first step to move towards modeling the full skeleton in VPlan,
start by wrapping IR blocks created during legacy skeleton creation in
VPIRBasicBlocks and hook them into the VPlan. This means the skeleton
CFG is represented in VPlan, just before execute. This allows moving
parts of skeleton creation into recipes in the VPBBs gradually.
Note that this allows retiring some manual DT updates, as this will be
handled automatically during VPlan execution.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 76 +++++++++++++------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 67 +++++++++-------
llvm/lib/Transforms/Vectorize/VPlan.h | 51 ++++---------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 -
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
.../Transforms/Vectorize/VPlanVerifier.cpp | 9 ---
.../Transforms/Vectorize/VPDomTreeTest.cpp | 9 ++-
.../Transforms/Vectorize/VPlanTest.cpp | 38 +++++++---
.../Vectorize/VPlanVerifierTest.cpp | 18 +++--
11 files changed, 167 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1c64bd2982d764..2dbee4331e238f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2426,6 +2426,26 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
return VectorTripCount;
}
+static void connectScalarPreheaderInVPlan(VPlan &Plan) {
+ VPBlockBase *VectorPH = Plan.getVectorPreheader();
+ VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
+ VPBlockUtils::disconnectBlocks(Plan.getEntry(), VectorPH);
+ VPBlockUtils::connectBlocks(PredVPB, ScalarPH);
+ VPBlockUtils::connectBlocks(PredVPB, VectorPH);
+}
+
+static void connectCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) {
+ VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ VPBlockBase *VectorPH = Plan.getVectorPreheader();
+ VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
+ VPBlockUtils::disconnectBlocks(PredVPB, VectorPH);
+ VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB);
+ VPBlockUtils::connectBlocks(PredVPB, CheckVPIRBB);
+ VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH);
+ VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH);
+}
+
void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
Value *Count = getTripCount();
// Reuse existing vector loop preheader for TC checks.
@@ -2511,13 +2531,14 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
"TC check is expected to dominate Bypass");
// Update dominator for Bypass & LoopExit (if needed).
- DT->changeImmediateDominator(Bypass, TCCheckBlock);
BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
LoopBypassBlocks.push_back(TCCheckBlock);
+
+ connectScalarPreheaderInVPlan(Plan);
}
BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
@@ -2534,6 +2555,8 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
"Should already be a bypass block due to iteration count check");
LoopBypassBlocks.push_back(SCEVCheckBlock);
AddedSafetyChecks = true;
+
+ connectCheckBlockInVPlan(Plan, SCEVCheckBlock);
return SCEVCheckBlock;
}
@@ -2570,6 +2593,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
AddedSafetyChecks = true;
+ connectCheckBlockInVPlan(Plan, MemCheckBlock);
return MemCheckBlock;
}
@@ -7648,10 +7672,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// 0. Generate SCEV-dependent code into the preheader, including TripCount,
// before making any changes to the CFG.
- if (!BestVPlan.getPreheader()->empty()) {
+ if (!BestVPlan.getEntry()->empty()) {
State.CFG.PrevBB = OrigLoop->getLoopPreheader();
State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator());
- BestVPlan.getPreheader()->execute(&State);
+ BestVPlan.getEntry()->execute(&State);
}
if (!ILV.getTripCount())
ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0)));
@@ -7859,8 +7883,6 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
DT->getNode(Bypass)->getIDom()) &&
"TC check is expected to dominate Bypass");
- // Update dominator for Bypass.
- DT->changeImmediateDominator(Bypass, TCCheckBlock);
LoopBypassBlocks.push_back(TCCheckBlock);
// Save the trip count so we don't have to regenerate it in the
@@ -7875,6 +7897,12 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
+ VPBlockBase *VectorPH = Plan.getVectorPreheader();
+ VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
+ if (PredVPB->getNumSuccessors() == 1)
+ connectScalarPreheaderInVPlan(Plan);
+ else
+ connectCheckBlockInVPlan(Plan, TCCheckBlock);
return TCCheckBlock;
}
@@ -7905,32 +7933,19 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopVectorPreHeader);
- DT->changeImmediateDominator(LoopVectorPreHeader,
- EPI.MainLoopIterationCountCheck);
-
EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
if (EPI.SCEVSafetyCheck)
EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
- if (EPI.MemSafetyCheck)
+ if (EPI.MemSafetyCheck) {
EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
-
- DT->changeImmediateDominator(
- VecEpilogueIterationCountCheck,
- VecEpilogueIterationCountCheck->getSinglePredecessor());
+ }
DT->changeImmediateDominator(LoopScalarPreHeader,
EPI.EpilogueIterationCountCheck);
- if (!Cost->requiresScalarEpilogue(EPI.EpilogueVF.isVector()))
- // If there is an epilogue which must run, there's no edge from the
- // middle block to exit blocks and thus no need to update the immediate
- // dominator of the exit blocks.
- DT->changeImmediateDominator(LoopExitBlock,
- EPI.EpilogueIterationCountCheck);
-
// Keep track of bypass blocks, as they feed start values to the induction and
// reduction phis in the scalar loop preheader.
if (EPI.SCEVSafetyCheck)
@@ -8033,6 +8048,20 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
}
ReplaceInstWithInst(Insert->getTerminator(), &BI);
LoopBypassBlocks.push_back(Insert);
+
+ // A new entry block has been created for the epilogue VPlan. Hook it in.
+ VPIRBasicBlock *NewEntry = VPIRBasicBlock::fromBasicBlock(Insert);
+ VPBasicBlock *OldEntry = Plan.getEntry();
+ VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
+ Plan.setEntry(NewEntry);
+ for (auto &R : make_early_inc_range(*NewEntry)) {
+ auto *VPIR = dyn_cast<VPIRInstruction>(&R);
+ if (!VPIR || !isa<PHINode>(VPIR->getInstruction()))
+ break;
+ VPIR->eraseFromParent();
+ }
+
+ connectScalarPreheaderInVPlan(Plan);
return Insert;
}
@@ -10256,7 +10285,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// should be removed once induction resume value creation is done
// directly in VPlan.
EpilogILV.setTripCount(MainILV.getTripCount());
- for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
+ for (auto &R : make_early_inc_range(*BestEpiPlan.getEntry())) {
auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpandR)
continue;
@@ -10316,8 +10345,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
}
- assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
- "DT not preserved correctly");
LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV,
DT, true, &ExpandedSCEVs);
++LoopsEpilogueVectorized;
@@ -10345,6 +10372,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
checkMixedPrecision(L, ORE);
}
+ assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
+ "DT not preserved correctly");
+
std::optional<MDNode *> RemainderLoopID =
makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
LLVMLoopVectorizeFollowupEpilogue});
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 54aae122877b98..7de8830bd866ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -170,9 +170,8 @@ VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
}
void VPBlockBase::setPlan(VPlan *ParentPlan) {
- assert(
- (ParentPlan->getEntry() == this || ParentPlan->getPreheader() == this) &&
- "Can only set plan on its entry or preheader block.");
+ assert(ParentPlan->getEntry() == this &&
+ "Can only set plan on its entry or preheader block.");
Plan = ParentPlan;
}
@@ -463,7 +462,6 @@ void VPIRBasicBlock::execute(VPTransformState *State) {
(getNumSuccessors() == 0 || isa<BranchInst>(IRBB->getTerminator())) &&
"other blocks must be terminated by a branch");
}
-
for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
BasicBlock *PredBB = State->CFG.VPBB2IRBB[PredVPBB];
@@ -851,9 +849,6 @@ VPlan::~VPlan() {
Block->dropAllReferences(&DummyValue);
VPBlockBase::deleteCFG(Entry);
-
- Preheader->dropAllReferences(&DummyValue);
- delete Preheader;
}
for (VPValue *VPV : VPLiveInsToFree)
delete VPV;
@@ -876,9 +871,10 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
VPIRBasicBlock *Entry =
VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader());
VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph");
+ VPBlockUtils::connectBlocks(Entry, VecPreheader);
VPIRBasicBlock *ScalarHeader =
VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader());
- auto Plan = std::make_unique<VPlan>(Entry, VecPreheader, ScalarHeader);
+ auto Plan = std::make_unique<VPlan>(Entry, ScalarHeader);
// Create SCEV and VPValue for the trip count.
@@ -1021,8 +1017,9 @@ void VPlan::execute(VPTransformState *State) {
BasicBlock *VectorPreHeader = State->CFG.PrevBB;
State->Builder.SetInsertPoint(VectorPreHeader->getTerminator());
- // Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
- cast<BranchInst>(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr);
+ replaceVPBBWithIRVPBB(
+ cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor()),
+ VectorPreHeader);
State->CFG.DTU.applyUpdates(
{{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}});
@@ -1049,8 +1046,10 @@ void VPlan::execute(VPTransformState *State) {
State->CFG.DTU.applyUpdates(
{{DominatorTree::Delete, ScalarPh, ScalarPh->getSingleSuccessor()}});
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Entry);
// Generate code in the loop pre-header and body.
- for (VPBlockBase *Block : vp_depth_first_shallow(Entry))
+ for (VPBlockBase *Block : make_range(RPOT.begin(), RPOT.end()))
Block->execute(State);
VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
@@ -1101,9 +1100,6 @@ void VPlan::execute(VPTransformState *State) {
}
State->CFG.DTU.flush();
- assert(State->CFG.DTU.getDomTree().verify(
- DominatorTree::VerificationLevel::Fast) &&
- "DT not preserved correctly");
}
InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) {
@@ -1156,12 +1152,10 @@ void VPlan::print(raw_ostream &O) const {
printLiveIns(O);
- if (!getPreheader()->empty()) {
- O << "\n";
- getPreheader()->print(O, "", SlotTracker);
- }
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<const VPBlockBase *>>
+ RPOT(getEntry());
- for (const VPBlockBase *Block : vp_depth_first_shallow(getEntry())) {
+ for (const VPBlockBase *Block : RPOT) {
O << '\n';
Block->print(O, "", SlotTracker);
}
@@ -1192,6 +1186,20 @@ std::string VPlan::getName() const {
return Out;
}
+VPRegionBlock *VPlan::getVectorLoopRegion() {
+ for (VPBlockBase *B : vp_depth_first_shallow(getEntry()))
+ if (auto *R = dyn_cast<VPRegionBlock>(B))
+ return R;
+ return nullptr;
+}
+
+const VPRegionBlock *VPlan::getVectorLoopRegion() const {
+ for (const VPBlockBase *B : vp_depth_first_shallow(getEntry()))
+ if (auto *R = dyn_cast<VPRegionBlock>(B))
+ return R;
+ return nullptr;
+}
+
LLVM_DUMP_METHOD
void VPlan::printDOT(raw_ostream &O) const {
VPlanPrinter Printer(O, *this);
@@ -1242,7 +1250,6 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry,
VPlan *VPlan::duplicate() {
// Clone blocks.
- VPBasicBlock *NewPreheader = Preheader->clone();
const auto &[NewEntry, __] = cloneFrom(Entry);
BasicBlock *ScalarHeaderIRBB = getScalarHeader()->getIRBasicBlock();
@@ -1252,8 +1259,7 @@ VPlan *VPlan::duplicate() {
return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB;
}));
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
- auto *NewPlan =
- new VPlan(NewPreheader, cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+ auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPValue *OldLiveIn : VPLiveInsToFree) {
Old2NewVPValues[OldLiveIn] =
@@ -1273,7 +1279,6 @@ VPlan *VPlan::duplicate() {
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
- remapOperands(Preheader, NewPreheader, Old2NewVPValues);
remapOperands(Entry, NewEntry, Old2NewVPValues);
// Initialize remaining fields of cloned VPlan.
@@ -1287,6 +1292,19 @@ VPlan *VPlan::duplicate() {
return NewPlan;
}
+VPBasicBlock *VPlan::getScalarPreheader() {
+ auto *MiddleVPBB =
+ cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
+ if (MiddleVPBB->getNumSuccessors() == 2) {
+ // Order is strict: first is the exit block, second is the scalar preheader.
+ return cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[1]);
+ }
+ if (auto *IRVPBB = dyn_cast<VPBasicBlock>(MiddleVPBB->getSingleSuccessor()))
+ return IRVPBB;
+
+ return nullptr;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
@@ -1325,8 +1343,6 @@ void VPlanPrinter::dump() {
OS << "edge [fontname=Courier, fontsize=30]\n";
OS << "compound=true\n";
- dumpBlock(Plan.getPreheader());
-
for (const VPBlockBase *Block : vp_depth_first_shallow(Plan.getEntry()))
dumpBlock(Block);
@@ -1587,7 +1603,6 @@ void VPSlotTracker::assignNames(const VPlan &Plan) {
assignName(Plan.BackedgeTakenCount);
for (VPValue *LI : Plan.VPLiveInsToFree)
assignName(LI);
- assignNames(Plan.getPreheader());
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<const VPBlockBase *>>
RPOT(VPBlockDeepTraversalWrapper<const VPBlockBase *>(Plan.getEntry()));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 035b9e66fd062a..c9393958521295 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3703,11 +3703,6 @@ class VPlan {
/// preheader of the vector loop.
VPBasicBlock *Entry;
- /// VPBasicBlock corresponding to the original preheader. Used to place
- /// VPExpandSCEV recipes for expressions used during skeleton creation and the
- /// rest of VPlan execution.
- VPBasicBlock *Preheader;
-
/// VPIRBasicBlock wrapping the header of the original scalar loop.
VPIRBasicBlock *ScalarHeader;
@@ -3752,36 +3747,27 @@ class VPlan {
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
public:
- /// Construct a VPlan with original preheader \p Preheader, trip count \p TC,
- /// \p Entry to the plan and with \p ScalarHeader wrapping the original header
- /// of the scalar loop. At the moment, \p Preheader and \p Entry need to be
- /// disconnected, as the bypass blocks between them are not yet modeled in
- /// VPlan.
- VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry,
- VPIRBasicBlock *ScalarHeader)
- : VPlan(Preheader, Entry, ScalarHeader) {
+ /// Construct a VPlan with \p Entry entering the plan and trip count \p TC.
+ VPlan(VPBasicBlock *Entry, VPValue *TC, VPIRBasicBlock *ScalarHeader)
+ : VPlan(Entry, ScalarHeader) {
TripCount = TC;
}
- /// Construct a VPlan with original preheader \p Preheader, \p Entry to
- /// the plan and with \p ScalarHeader wrapping the original header of the
- /// scalar loop. At the moment, \p Preheader and \p Entry need to be
- /// disconnected, as the bypass blocks between them are not yet modeled in
- /// VPlan.
- VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry,
- VPIRBasicBlock *ScalarHeader)
- : Entry(Entry), Preheader(Preheader), ScalarHeader(ScalarHeader) {
+ /// Construct a VPlan with \p Entry to the plan.
+ VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
+ : Entry(Entry), ScalarHeader(ScalarHeader) {
Entry->setPlan(this);
- Preheader->setPlan(this);
- assert(Preheader->getNumSuccessors() == 0 &&
- Preheader->getNumPredecessors() == 0 &&
- "preheader must be disconnected");
assert(ScalarHeader->getNumSuccessors() == 0 &&
"scalar header must be a leaf node");
}
~VPlan();
+ void setEntry(VPBasicBlock *VPBB) {
+ Entry = VPBB;
+ VPBB->setPlan(this);
+ }
+
/// Create initial VPlan, having an "entry" VPBasicBlock (wrapping
/// original scalar pre-header ) which contains SCEV expansions that need
/// to happen before the CFG is modified; a VPBasicBlock for the vector
@@ -3932,12 +3918,9 @@ class VPlan {
#endif
/// Returns the VPRegionBlock of the vector loop.
- VPRegionBlock *getVectorLoopRegion() {
- return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
- }
- const VPRegionBlock *getVectorLoopRegion() const {
- return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
- }
+ VPRegionBlock *getVectorLoopRegion();
+
+ const VPRegionBlock *getVectorLoopRegion() const;
/// Returns the preheader of the vector loop region.
VPBasicBlock *getVectorPreheader() {
@@ -3963,13 +3946,11 @@ class VPlan {
SCEVToExpansion[S] = V;
}
- /// \return The block corresponding to the original preheader.
- VPBasicBlock *getPreheader() { return Preheader; }
- const VPBasicBlock *getPreheader() const { return Preheader; }
-
/// Clone the current VPlan, update all VPValues of the new VPlan and cloned
/// recipes to refer to the clones, and return it.
VPlan *duplicate();
+
+ VPBasicBlock *getScalarPreheader();
};
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6254ea15191819..39e06736cfb320 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -627,11 +627,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
Builder.CreatePHI(IncomingFromOtherPreds->getType(), 2, Name);
BasicBlock *VPlanPred =
State.CFG
- .VPBB2IRBB[cast<VPBasicBlock>(getParent()->getSinglePredecessor())];
+ .VPBB2IRBB[cast<VPBasicBlock>(getParent()->getPredecessors()[0])];
NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);
for (auto *OtherPred : predecessors(Builder.GetInsertBlock())) {
- assert(OtherPred != VPlanPred &&
- "VPlan predecessors should not be connected yet");
+ if (OtherPred == VPlanPred)
+ continue;
NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);
}
return NewPhi;
@@ -3235,13 +3235,17 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
void VPExpandSCEVRecipe::execute(VPTransformState &State) {
assert(!State.Lane && "cannot be used in per-lane");
+ if (State.ExpandedSCEVs.contains(Expr)) {
+ State.Builder.SetInsertPoint(State.CFG.VPBB2IRBB[getParent()]);
+ return;
+ }
+
const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
SCEVExpander Exp(SE, DL, "induction");
-
Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
&*State.Builder.GetInsertPoint());
- assert(!State.ExpandedSCEVs.contains(Expr) &&
- "Same SCEV expanded multiple times");
+ /* assert(!State.ExpandedSCEVs.contains(Expr) &&*/
+ /*"Same SCEV expanded multiple times");*/
State.ExpandedSCEVs[Expr] = Res;
State.set(this, Res, VPLane(0));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a4f0df17f5832f..33ef10dbefc507 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -381,6 +381,8 @@ static bool mergeBlocksIntoPredecessors(VPlan &Plan) {
dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
if (!PredVPBB || PredVPBB->getNumSuccessors() != 1)
continue;
+ if (isa<VPIRBasicBlock>(PredVPBB))
+ continue;
WorkList.push_back(VPBB);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index dd005682203b75..ae53774bff98db 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -412,8 +412,6 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) {
UnrollState Unroller(Plan, UF, Ctx);
- Unroller.unrollBlock(Plan.getPreheader());
-
// Iterate over all blocks in the plan starting from Entry, and unroll
// recipes inside them. This includes the vector preheader and middle blocks,
// which may set up or post-process per-part values.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4621c28b051298..e40af3e2e3d30a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -34,7 +34,7 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
Expanded = Plan.getOrAddLiveIn(E->getValue());
else {
Expanded = new VPExpandSCEVRecipe(Expr, SE);
- Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe());
+ Plan.getEntry()->appendRecipe(Expanded->getDefiningRecipe());
}
Plan.addSCEVExpansion(Expr, Expanded);
return Expanded;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 8bdb3133243582..fc6b61491f8b5e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -195,15 +195,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
RecipeNumbering[&R] = Cnt++;
for (const VPRecipeBase &R : *VPBB) {
- if (isa<VPIRInstruction>(&R) ^ isa<VPIRBasicBlock>(VPBB)) {
- errs() << "VPIRInstructions ";
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- R.dump();
- errs() << " ";
-#endif
- errs() << "not in a VPIRBasicBlock!\n";
- return false;
- }
for (const VPValue *V : R.definedValues()) {
for (const VPUser *U : V->users()) {
auto *UI = dyn_cast<VPRecipeBase>(U);
diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
index 86182ccae0b55b..604f87b634bba6 100644
--- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
@@ -44,8 +44,9 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
@@ -124,8 +125,9 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
@@ -206,8 +208,9 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 4a8615cc086b08..81f1707aaf7fba 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -259,8 +259,9 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, VPBB2->getPlan());
@@ -281,8 +282,9 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, R1->getPlan());
@@ -313,8 +315,9 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, R1->getPlan());
@@ -359,8 +362,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
}
{
@@ -461,8 +465,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
}
{
@@ -546,8 +551,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
}
{
@@ -596,8 +602,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
}
{
@@ -690,8 +697,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
}
delete ScalarHeader;
}
@@ -734,7 +742,8 @@ TEST(VPBasicBlockTest, print) {
auto *ScalarHeader = BasicBlock::Create(C, "");
// FIXME: This looks wrong.
auto ScalarHeaderVPBB = std::make_unique<VPIRBasicBlock>(ScalarHeader);
- VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB.get());
+ VPBlockUtils::connectBlocks(VPBB0, VPBB1);
+ VPlan Plan(VPBB0, TC, ScalarHeaderVPBB.get());
std::string FullDump;
raw_string_ostream OS(FullDump);
Plan.printDOT(OS);
@@ -820,8 +829,9 @@ TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPBB0, VPBB1);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, TC, ScalarHeaderVPBB);
Plan.setName("TestPlan");
Plan.addVF(ElementCount::getFixed(4));
@@ -1291,11 +1301,13 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
TEST(VPRecipeTest, dumpRecipeInPlan) {
VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
VPBasicBlock *VPBB1 = new VPBasicBlock();
+ VPBlockUtils::connectBlocks(VPBB0, VPBB1);
+
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, ScalarHeaderVPBB);
IntegerType *Int32 = IntegerType::get(C, 32);
auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32),
@@ -1363,11 +1375,13 @@ TEST(VPRecipeTest, dumpRecipeInPlan) {
TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
VPBasicBlock *VPBB1 = new VPBasicBlock();
+ VPBlockUtils::connectBlocks(VPBB0, VPBB1);
+
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, ScalarHeaderVPBB);
IntegerType *Int32 = IntegerType::get(C, 32);
auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32),
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index edb3f8a2952942..2c7566c973c4e1 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -32,8 +32,9 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -69,8 +70,9 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -116,8 +118,9 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
auto TC = std::make_unique<VPValue>();
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -157,8 +160,9 @@ TEST(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -199,8 +203,9 @@ TEST(VPVerifierTest, DuplicateSuccessorsInsideRegion) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -233,8 +238,9 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
+ VPBlockUtils::connectBlocks(VPPH, R1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
>From b87cf14c957352373feda487be5523d6e0d729d5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 7 Nov 2024 13:13:15 +0000
Subject: [PATCH 02/10] !fixup address comments, thanks!
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 65b11ed551fe2b..d6d8b28e96815c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2429,6 +2429,10 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
return VectorTripCount;
}
+/// Helper to connect both the vector and scalar preheaders to the vector
+/// preheader's predecessor. This is used when adjusting \p Plan during skeleton
+/// creation, i.e. adjusting the plan after introducing an initial runtime
+/// check.
static void connectScalarPreheaderInVPlan(VPlan &Plan) {
VPBlockBase *VectorPH = Plan.getVectorPreheader();
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
>From 599e690f42637f6955f1d338b77229e869766d3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 7 Nov 2024 21:40:24 +0000
Subject: [PATCH 03/10] !fixup address latest comments, thanks!
---
.../Transforms/Vectorize/LoopVectorize.cpp | 34 +++++++++----------
llvm/lib/Transforms/Vectorize/VPlan.h | 18 +++++++---
2 files changed, 30 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d6d8b28e96815c..f1fbaf88ca6a50 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2429,28 +2429,29 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
return VectorTripCount;
}
-/// Helper to connect both the vector and scalar preheaders to the vector
-/// preheader's predecessor. This is used when adjusting \p Plan during skeleton
+/// Helper to connect both the vector and scalar preheaders to the Plan's
+/// entry. This is used when adjusting \p Plan during skeleton
/// creation, i.e. adjusting the plan after introducing an initial runtime
/// check.
static void connectScalarPreheaderInVPlan(VPlan &Plan) {
VPBlockBase *VectorPH = Plan.getVectorPreheader();
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
- VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
- VPBlockUtils::disconnectBlocks(Plan.getEntry(), VectorPH);
- VPBlockUtils::connectBlocks(PredVPB, ScalarPH);
- VPBlockUtils::connectBlocks(PredVPB, VectorPH);
+ VPBlockBase *PredVPB = Plan.getEntry();
+ VPBlockUtils::connectBlocks(PredVPB, ScalarPH, -1, 0);
+ VPBlockUtils::connectBlocks(PredVPB, VectorPH, 0, -1);
}
-static void connectCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) {
+/// Introduces a new VPIRBasicBlock for \p CheckIRBB to \p Plan between the
+/// vector preheader and its predecessor, also connecting to the scalar
+/// preheader.
+static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) {
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
VPBlockBase *VectorPH = Plan.getVectorPreheader();
- VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
- VPBlockUtils::disconnectBlocks(PredVPB, VectorPH);
+ VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor();
VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB);
- VPBlockUtils::connectBlocks(PredVPB, CheckVPIRBB);
+ VPBlockUtils::connectBlocks(PreVectorPH, CheckVPIRBB, -1, 1);
VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH);
- VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH);
+ VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH, 0, -1);
}
void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
@@ -2537,7 +2538,6 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
DT->getNode(Bypass)->getIDom()) &&
"TC check is expected to dominate Bypass");
- // Update dominator for Bypass & LoopExit (if needed).
BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
@@ -2545,6 +2545,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
LoopBypassBlocks.push_back(TCCheckBlock);
+ // TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here.
connectScalarPreheaderInVPlan(Plan);
}
@@ -2563,7 +2564,7 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
LoopBypassBlocks.push_back(SCEVCheckBlock);
AddedSafetyChecks = true;
- connectCheckBlockInVPlan(Plan, SCEVCheckBlock);
+ introduceCheckBlockInVPlan(Plan, SCEVCheckBlock);
return SCEVCheckBlock;
}
@@ -2600,7 +2601,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
AddedSafetyChecks = true;
- connectCheckBlockInVPlan(Plan, MemCheckBlock);
+ introduceCheckBlockInVPlan(Plan, MemCheckBlock);
return MemCheckBlock;
}
@@ -7909,7 +7910,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
if (PredVPB->getNumSuccessors() == 1)
connectScalarPreheaderInVPlan(Plan);
else
- connectCheckBlockInVPlan(Plan, TCCheckBlock);
+ introduceCheckBlockInVPlan(Plan, TCCheckBlock);
return TCCheckBlock;
}
@@ -7946,10 +7947,9 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
if (EPI.SCEVSafetyCheck)
EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
- if (EPI.MemSafetyCheck) {
+ if (EPI.MemSafetyCheck)
EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
- }
DT->changeImmediateDominator(LoopScalarPreHeader,
EPI.EpilogueIterationCountCheck);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5414b30db595fa..46ba6328f0e1a4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1410,7 +1410,7 @@ class VPIRInstruction : public VPRecipeBase {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- Instruction &getInstruction() { return I; }
+ Instruction &getInstruction() const { return I; }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -4098,13 +4098,21 @@ class VPBlockUtils {
/// the successors of \p From and \p From to the predecessors of \p To. Both
/// VPBlockBases must have the same parent, which can be null. Both
/// VPBlockBases can be already connected to other VPBlockBases.
- static void connectBlocks(VPBlockBase *From, VPBlockBase *To) {
+ static void connectBlocks(VPBlockBase *From, VPBlockBase *To,
+ unsigned PredIdx = -1u, unsigned SuccIdx = -1u) {
assert((From->getParent() == To->getParent()) &&
"Can't connect two block with different parents");
- assert(From->getNumSuccessors() < 2 &&
+ assert((SuccIdx != -1u || From->getNumSuccessors() < 2) &&
"Blocks can't have more than two successors.");
- From->appendSuccessor(To);
- To->appendPredecessor(From);
+ if (SuccIdx == -1u)
+ From->appendSuccessor(To);
+ else
+ From->getSuccessors()[SuccIdx] = To;
+
+ if (PredIdx == -1u)
+ To->appendPredecessor(From);
+ else
+ To->getPredecessors()[PredIdx] = From;
}
/// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
>From 1a77b551357b630e65b8fcc8cba996cde7cb1505 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 9 Nov 2024 11:24:29 +0000
Subject: [PATCH 04/10] [VPlan] Add PredIdx and SuccIdx arguments to
connectBlocks (NFC).
---
llvm/lib/Transforms/Vectorize/VPlan.h | 16 ++++++++++++----
.../lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 ++---
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 18f5f13073aa63..2a4bcd30758468 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4117,13 +4117,21 @@ class VPBlockUtils {
/// the successors of \p From and \p From to the predecessors of \p To. Both
/// VPBlockBases must have the same parent, which can be null. Both
/// VPBlockBases can be already connected to other VPBlockBases.
- static void connectBlocks(VPBlockBase *From, VPBlockBase *To) {
+ static void connectBlocks(VPBlockBase *From, VPBlockBase *To,
+ unsigned PredIdx = -1u, unsigned SuccIdx = -1u) {
assert((From->getParent() == To->getParent()) &&
"Can't connect two block with different parents");
- assert(From->getNumSuccessors() < 2 &&
+ assert((SuccIdx != -1u || From->getNumSuccessors() < 2) &&
"Blocks can't have more than two successors.");
- From->appendSuccessor(To);
- To->appendPredecessor(From);
+ if (SuccIdx == -1u)
+ From->appendSuccessor(To);
+ else
+ From->getSuccessors()[SuccIdx] = To;
+
+ if (PredIdx == -1u)
+ To->appendPredecessor(From);
+ else
+ To->getPredecessors()[PredIdx] = From;
}
/// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ea8845eaa75d4d..70163aacb19d9d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -360,9 +360,8 @@ static void addReplicateRegions(VPlan &Plan) {
// Record predicated instructions for above packing optimizations.
VPBlockBase *Region = createReplicateRegion(RepR, Plan);
Region->setParent(CurrentBlock->getParent());
- VPBlockUtils::disconnectBlocks(CurrentBlock, SplitBlock);
- VPBlockUtils::connectBlocks(CurrentBlock, Region);
- VPBlockUtils::connectBlocks(Region, SplitBlock);
+ VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0);
+ VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1);
}
}
>From a2137b4dea6380320668cec61f5ef9474bb0cbbd Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 9 Nov 2024 13:46:48 +0000
Subject: [PATCH 05/10] [VPlan] Add insertOnEdge (NFC).
---
llvm/lib/Transforms/Vectorize/VPlan.h | 15 +++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2a4bcd30758468..4038e72829088f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4173,6 +4173,21 @@ class VPBlockUtils {
return cast<BlockTy>(&Block);
});
}
+
+ static void insertOnEdge(VPBlockBase *From, VPBlockBase *To,
+ VPBlockBase *BlockPtr) {
+ unsigned SuccIdx =
+ std::distance(From->getSuccessors().begin(),
+ find_if(From->getSuccessors(),
+ [To](VPBlockBase *Succ) { return To == Succ; }));
+ unsigned PredIx =
+ std::distance(To->getPredecessors().begin(),
+ find_if(To->getPredecessors(), [From](VPBlockBase *Pred) {
+ return From == Pred;
+ }));
+ VPBlockUtils::connectBlocks(From, BlockPtr, -1, SuccIdx);
+ VPBlockUtils::connectBlocks(BlockPtr, To, PredIx, -1);
+ }
};
class VPInterleavedAccessInfo {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 70163aacb19d9d..b9ab8a8fe60107 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -360,8 +360,7 @@ static void addReplicateRegions(VPlan &Plan) {
// Record predicated instructions for above packing optimizations.
VPBlockBase *Region = createReplicateRegion(RepR, Plan);
Region->setParent(CurrentBlock->getParent());
- VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0);
- VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1);
+ VPBlockUtils::insertOnEdge(CurrentBlock, SplitBlock, Region);
}
}
>From be2c3a658f081bba28c1d6f490c149e0d16c772c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 9 Nov 2024 14:04:40 +0000
Subject: [PATCH 06/10] !fixup use insertOnEdge.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 +--
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 05d48685afffff..a8ab3321aa15ef 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2449,9 +2449,8 @@ static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) {
VPBlockBase *VectorPH = Plan.getVectorPreheader();
VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor();
VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB);
- VPBlockUtils::connectBlocks(PreVectorPH, CheckVPIRBB, -1, 1);
VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH);
- VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH, 0, -1);
+ VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB);
}
void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index b4e2b298196b61..13357738eea6d2 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -743,7 +743,7 @@ TEST(VPBasicBlockTest, print) {
auto * ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
VPBlockUtils::connectBlocks(VPBB0, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, TC, ScalarHeaderVPBB);
std::string FullDump;
raw_string_ostream OS(FullDump);
Plan.printDOT(OS);
>From 466b3931c842b19ec1473146bb89f13725db4ca9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 9 Nov 2024 19:15:33 +0000
Subject: [PATCH 07/10] [VPlan] Add insertOnEdge to VPBlockUtils (NFC).
Add a new helper to insert a new VPBlockBase on an edge between 2
blocks. Suggested in https://github.com/llvm/llvm-project/pull/114292
and also useful for some existing code.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 18 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +--
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0770ff032aa6a0..abfe97b4ab55b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4177,6 +4177,24 @@ class VPBlockUtils {
return cast<BlockTy>(&Block);
});
}
+
+ /// Inserts \p BlockPtr on the edge between \p From and \p To. That is, update
+ /// \p From's successor to \p To to point to \p BlockPtr and \p To's
+ /// predecessor from \p From to \p BlockPtr. \p From and \p To are added to \p
+ /// BlockPtr's predecessors and successors respectively. There must be a
+ /// single edge between \p From and \p To.
+ static void insertOnEdge(VPBlockBase *From, VPBlockBase *To,
+ VPBlockBase *BlockPtr) {
+ auto &Successors = From->getSuccessors();
+ auto &Predecessors = To->getPredecessors();
+ assert(count(Successors, To) == 1 && count(Predecessors, From) == 1 &&
+ "must have single between From and To");
+ unsigned SuccIdx = std::distance(Successors.begin(), find(Successors, To));
+ unsigned PredIx =
+ std::distance(Predecessors.begin(), find(Predecessors, From));
+ VPBlockUtils::connectBlocks(From, BlockPtr, -1, SuccIdx);
+ VPBlockUtils::connectBlocks(BlockPtr, To, PredIx, -1);
+ }
};
class VPInterleavedAccessInfo {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 70163aacb19d9d..b9ab8a8fe60107 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -360,8 +360,7 @@ static void addReplicateRegions(VPlan &Plan) {
// Record predicated instructions for above packing optimizations.
VPBlockBase *Region = createReplicateRegion(RepR, Plan);
Region->setParent(CurrentBlock->getParent());
- VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0);
- VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1);
+ VPBlockUtils::insertOnEdge(CurrentBlock, SplitBlock, Region);
}
}
>From 79964516f2c520e7fef255743252e971078f16d2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 9 Nov 2024 19:21:49 +0000
Subject: [PATCH 08/10] !fixup cleanup after merge.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 -
llvm/lib/Transforms/Vectorize/VPlan.cpp | 5 +
llvm/lib/Transforms/Vectorize/VPlan.h | 2 -
.../AArch64/sve-tail-folding-forced.ll | 2 +-
.../LoopVectorize/AArch64/sve-widen-gep.ll | 3 +
.../AArch64/sve2-histcnt-vplan.ll | 18 ++-
.../AArch64/synthesize-mask-for-call.ll | 54 ++++---
.../widen-call-with-intrinsic-or-libfunc.ll | 18 ++-
.../PowerPC/vplan-force-tail-with-evl.ll | 6 +
.../RISCV/riscv-vector-reverse.ll | 136 ++---------------
.../RISCV/vplan-vp-intrinsics-reduction.ll | 37 ++---
.../RISCV/vplan-vp-select-intrinsics.ll | 23 ++-
.../first-order-recurrence-chains-vplan.ll | 36 +++--
...-order-recurrence-sink-replicate-region.ll | 55 ++++---
.../Transforms/LoopVectorize/icmp-uniforms.ll | 3 +
.../interleave-and-scalarize-only.ll | 19 ++-
.../LoopVectorize/vplan-dot-printing.ll | 3 +-
.../LoopVectorize/vplan-iv-transforms.ll | 9 +-
.../LoopVectorize/vplan-predicate-switch.ll | 28 ++--
.../vplan-printing-before-execute.ll | 38 +++--
.../vplan-printing-outer-loop.ll | 9 +-
.../LoopVectorize/vplan-printing.ll | 144 +++++++++++-------
.../vplan-sink-scalars-and-merge-vf1.ll | 9 +-
.../vplan-sink-scalars-and-merge.ll | 44 ++++--
.../vplan-unused-interleave-group.ll | 3 +
25 files changed, 376 insertions(+), 333 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a8ab3321aa15ef..bfb4fc0d3506da 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7671,11 +7671,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
OrigLoop->getHeader()->getContext());
VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
- LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF
- << ", UF=" << BestUF << '\n');
- BestVPlan.setName("Final VPlan");
- LLVM_DEBUG(BestVPlan.dump());
-
// Perform the actual loop transformation.
VPTransformState State(BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f4f19f51a9036b..31436c070a314f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1023,6 +1023,11 @@ void VPlan::execute(VPTransformState *State) {
replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh);
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
+ LLVM_DEBUG(dbgs() << "Executing best plan with VF=" <<State->VF
+ << ", UF=" << getUF()<< '\n');
+ setName("Final VPlan");
+ LLVM_DEBUG(dump());
+
// Disconnect the middle block from its single successor (the scalar loop
// header) in both the CFG and DT. The branch will be recreated during VPlan
// execution.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c69a3be6229307..d84ec63c6e3e6b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4104,8 +4104,6 @@ class VPBlockUtils {
unsigned PredIdx = -1u, unsigned SuccIdx = -1u) {
assert((From->getParent() == To->getParent()) &&
"Can't connect two block with different parents");
- assert((PredIdx != -1u || To->getNumPredecessors() < 2) &&
- "Blocks can't have more than two predecessors.");
if (SuccIdx == -1u)
From->appendSuccessor(To);
else
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index 0b3f28e8db5c4d..928b86d5c70b1f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -15,7 +15,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: ir-bb<entry>:
; VPLANS-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 umax %n)
-; VPLANS-NEXT: No successors
+; VPLANS-NEXT: Successor(s): vector.ph
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 6ec9eb849dd52a..cd1c32fb56fa7b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -13,6 +13,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%N> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
index 6257d3325f9796..3ef99ff496a687 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -16,6 +16,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -41,9 +44,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]]
; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -51,6 +51,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;; Check that the vectorized plan contains a histogram recipe instead.
@@ -59,6 +62,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -83,9 +89,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]]
; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -93,6 +96,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index af1c63bc405a38..8ac46fe7687d24 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -14,6 +14,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -39,9 +42,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -49,6 +49,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -56,6 +59,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -81,9 +87,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -91,6 +94,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;; If we have a masked variant at one VF and an unmasked variant at a different
@@ -103,6 +109,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -128,9 +137,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -138,6 +144,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -145,6 +154,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -170,9 +182,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -180,6 +189,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;; If we have two variants at different VFs, neither of which are masked, we
@@ -191,6 +203,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -216,9 +231,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -226,6 +238,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -233,6 +248,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -258,9 +276,6 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -268,6 +283,9 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
define void @test_v4_v4m(ptr noalias %a, ptr readonly %b) #3 {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index afc2fd5a049ad1..648f6e874abbe0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -12,6 +12,9 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -37,9 +40,6 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -47,6 +47,9 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
@@ -54,6 +57,9 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -79,9 +85,6 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -89,6 +92,9 @@ target triple = "arm64-apple-ios"
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
;
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
index a0696b3204dbd4..cd1d18aad8361d 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll
@@ -14,6 +14,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%N> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -77,6 +80,9 @@ define void @safe_dep(ptr %p) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<512> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index a38835f5613fd8..ec23fd9b83392b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -62,7 +62,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: ir-bb<for.body.preheader>:
; CHECK-NEXT: IR %0 = zext i32 %n to i64
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -92,9 +92,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -103,6 +100,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
@@ -139,68 +139,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Loop cost is 32
; CHECK-NEXT: LV: IC is 1
; CHECK-NEXT: LV: VF is vscale x 4
-; CHECK-NEXT: LV: Not Interleaving.
-; CHECK-NEXT: LV: Interleaving is not beneficial.
-; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in <stdin>
-; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop
-; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1
-; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.body.preheader>:
-; CHECK-NEXT: IR %0 = zext i32 %n to i64
-; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT: WIDEN ir<%13> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%add9> = add ir<%13>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
-; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
-; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<for.body>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.body>:
-; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
-; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Interleaving disabled by the pass manager
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Vectorizing: innermost loop.
-; CHECK-EMPTY:
;
entry:
%cmp7 = icmp sgt i32 %n, 0
@@ -281,7 +219,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: ir-bb<for.body.preheader>:
; CHECK-NEXT: IR %0 = zext i32 %n to i64
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -311,9 +249,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -322,6 +257,9 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
@@ -362,63 +300,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Interleaving is not beneficial.
; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in <stdin>
; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop
-; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1
-; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.body.preheader>:
-; CHECK-NEXT: IR %0 = zext i32 %n to i64
-; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
-; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
-; CHECK-NEXT: WIDEN ir<%13> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00>
-; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]>
-; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%conv1>
-; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
-; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<for.body>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.body>:
-; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ]
-; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Interleaving disabled by the pass manager
-; CHECK-NEXT: LV: Loop does not require scalar epilogue
-; CHECK-NEXT: LV: Vectorizing: innermost loop.
+; CHECK: Executing best plan with VF=vscale x 4, UF=1
;
entry:
%cmp7 = icmp sgt i32 %n, 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 414f39d557044a..ac3dfe8307fa2e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -29,7 +29,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
; IF-EVL-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count
; IF-EVL-OUTLOOP-EMPTY:
-; IF-EVL-OUTLOOP: vector.ph:
+; IF-EVL-OUTLOOP-NEXT: ir-bb<entry>:
+; IF-EVL-OUTLOOP-NEXT: Successor(s): vector.ph
+; IF-EVL-OUTLOOP-EMPTY:
+; IF-EVL-OUTLOOP-NEXT: vector.ph:
; IF-EVL-OUTLOOP-NEXT: Successor(s): vector loop
; IF-EVL-OUTLOOP-EMPTY:
; IF-EVL-OUTLOOP-NEXT: <x1> vector loop: {
@@ -59,10 +62,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-cond ir<true>
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; IF-EVL-OUTLOOP-EMPTY:
-; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
-; IF-EVL-OUTLOOP-NEXT: No successors
-; IF-EVL-OUTLOOP-EMPTY:
; IF-EVL-OUTLOOP-NEXT: scalar.ph:
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -72,6 +71,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
; IF-EVL-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n
; IF-EVL-OUTLOOP-NEXT: No successors
+; IF-EVL-OUTLOOP-EMPTY:
+; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
+; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; IF-EVL-OUTLOOP-NEXT: No successors
; IF-EVL-OUTLOOP-NEXT: }
;
@@ -109,10 +112,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: EMIT branch-on-cond ir<true>
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; IF-EVL-INLOOP-EMPTY:
-; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
-; IF-EVL-INLOOP-NEXT: No successors
-; IF-EVL-INLOOP-EMPTY:
; IF-EVL-INLOOP-NEXT: scalar.ph:
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -122,6 +121,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
; IF-EVL-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n
; IF-EVL-INLOOP-NEXT: No successors
+; IF-EVL-INLOOP-EMPTY:
+; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
+; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; IF-EVL-INLOOP-NEXT: No successors
; IF-EVL-INLOOP-NEXT: }
;
@@ -155,10 +158,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; NO-VP-OUTLOOP-EMPTY:
-; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
-; NO-VP-OUTLOOP-NEXT: No successors
-; NO-VP-OUTLOOP-EMPTY:
; NO-VP-OUTLOOP-NEXT: scalar.ph:
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -168,6 +167,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
; NO-VP-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n
; NO-VP-OUTLOOP-NEXT: No successors
+; NO-VP-OUTLOOP-EMPTY:
+; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
+; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; NO-VP-OUTLOOP-NEXT: No successors
; NO-VP-OUTLOOP-NEXT: }
;
@@ -201,10 +204,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; NO-VP-INLOOP-EMPTY:
-; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
-; NO-VP-INLOOP-NEXT: No successors
-; NO-VP-INLOOP-EMPTY:
; NO-VP-INLOOP-NEXT: scalar.ph:
; NO-VP-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -214,6 +213,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]
; NO-VP-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n
; NO-VP-INLOOP-NEXT: No successors
+; NO-VP-INLOOP-EMPTY:
+; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
+; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; NO-VP-INLOOP-NEXT: No successors
; NO-VP-INLOOP-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 6d6cfb5e9d18ed..523cf1bd798b68 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -7,11 +7,22 @@
define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
- ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
- ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
+ ; IF-EVL-NEXT: Live-in ir<%8> = VF * UF
+ ; IF-EVL-NEXT: Live-in ir<%n.vec> = vector-trip-count
; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
- ; IF-EVL: vector.ph:
+ ; IF-EVL: ir-bb<entry>:
+ ; IF-EVL-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
+
+ ; IF-EVL: ir-bb<vector.ph>:
+ ; IF-EVL-NEXT: IR %4 = call i64 @llvm.vscale.i64()
+ ; IF-EVL-NEXT: IR %5 = mul i64 %4, 4
+ ; IF-EVL-NEXT: IR %6 = sub i64 %5, 1
+ ; IF-EVL-NEXT: IR %n.rnd.up = add i64 %N, %6
+ ; IF-EVL-NEXT: IR %n.mod.vf = urem i64 %n.rnd.up, %5
+ ; IF-EVL-NEXT: IR %n.vec = sub i64 %n.rnd.up, %n.mod.vf
+ ; IF-EVL-NEXT: IR %7 = call i64 @llvm.vscale.i64()
+ ; IF-EVL-NEXT: IR %8 = mul i64 %7, 4
; IF-EVL-NEXT: Successor(s): vector loop
; IF-EVL: <x1> vector loop: {
@@ -29,15 +40,15 @@
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%10>, ir<%11>, vp<[[EVL]]>)
; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
- ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+ ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, ir<%8>
+ ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<%n.vec>
; IF-EVL-NEXT: No successors
; IF-EVL-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index bcacfb358ec05a..49e5220c46c9c9 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -9,6 +9,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -39,9 +42,6 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
@@ -53,6 +53,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -81,6 +84,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -115,9 +121,6 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33>
@@ -131,6 +134,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -165,6 +171,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<4098> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: WIDEN ir<%for.x.next> = mul ir<%x>, ir<2>
; CHECK-NEXT: Successor(s): vector loop
@@ -195,9 +204,6 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]>
; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<ret>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
@@ -208,6 +214,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]>)
; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1)
; CHECK: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -237,6 +246,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<4098> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -269,9 +281,6 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]>
; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<ret>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0>
@@ -282,6 +291,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]>)
; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1)
; CHECK: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<ret>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 8ae538cf63986b..8032decd2271d7 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -15,6 +15,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<20001> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -77,9 +80,6 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -89,6 +89,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %ec = icmp eq i32 %iv.next, 20001
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -121,6 +124,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<20001> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: Successor(s): vector loop
@@ -164,9 +170,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -176,6 +179,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %ec = icmp eq i32 %iv.next, 20001
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -205,6 +211,9 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<20001> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: Successor(s): vector loop
@@ -234,10 +243,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]>)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234>
@@ -249,6 +254,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ]
; CHECK: IR %ec = icmp eq i32 %iv.next, 20001
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]>)
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -282,6 +291,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<20001> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -347,9 +359,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -359,6 +368,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %ec = icmp eq i32 %iv.next, 20001
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -397,7 +409,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 smax (1 + (sext i8 %y to i32))<nsw>)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
@@ -444,9 +456,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -456,6 +465,9 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %C = icmp sgt i32 %iv.next, %recur.next
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -487,6 +499,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -530,9 +545,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -542,6 +554,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<[[RESUME_1_P]]>)
; CHECK: IR %ec = icmp ugt i64 %iv, 3
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index c71ff2b9fa4c47..bc698b6710e99d 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -43,6 +43,9 @@ for.end:
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<14> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index a71666d8c3167a..bfdadd17188204 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -11,7 +11,7 @@
; DBG-EMPTY:
; DBG-NEXT: ir-bb<entry>:
; DBG-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1000 + (-1 * %start))
-; DBG-NEXT: No successors
+; DBG-NEXT: Successor(s): vector.ph
; DBG-EMPTY:
; DBG-NEXT: vector.ph:
; DBG-NEXT: Successor(s): vector loop
@@ -72,6 +72,9 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; DBG-NEXT: Live-in ir<1000> = original trip-count
; DBG-EMPTY:
+; DBG-NEXT: ir-bb<entry>:
+; DBG-NEXT: Successor(s): vector.ph
+; DBG-EMPTY:
; DBG-NEXT: vector.ph:
; DBG-NEXT: Successor(s): vector loop
; DBG-EMPTY:
@@ -112,9 +115,6 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; DBG-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; DBG-EMPTY:
-; DBG-NEXT: ir-bb<exit>:
-; DBG-NEXT: No successors
-; DBG-EMPTY:
; DBG-NEXT: scalar.ph:
; DBG-NEXT: Successor(s): ir-bb<loop.header>
; DBG-EMPTY:
@@ -123,6 +123,9 @@ declare i32 @llvm.smin.i32(i32, i32)
; DBG-NEXT: IR %d = phi i1 [ false, %entry ], [ %d.next, %loop.latch ]
; DBG-NEXT: IR %d.next = xor i1 %d, true
; DBG-NEXT: No successors
+; DBG-EMPTY:
+; DBG-NEXT: ir-bb<exit>:
+; DBG-NEXT: No successors
; DBG-NEXT: }
define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
@@ -192,7 +195,7 @@ exit:
; DBG-EMPTY:
; DBG-NEXT: ir-bb<entry>:
; DBG-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 (1 smax %n) to i64)
-; DBG-NEXT: No successors
+; DBG-NEXT: Successor(s): vector.ph
; DBG-EMPTY:
; DBG-NEXT: vector.ph:
; DBG-NEXT: SCALAR-CAST vp<[[CAST:%.+]]> = trunc ir<1> to i32
@@ -218,9 +221,6 @@ exit:
; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; DBG-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; DBG-EMPTY:
-; DBG-NEXT: ir-bb<exit>:
-; DBG-NEXT: No successors
-; DBG-EMPTY:
; DBG-NEXT: scalar.ph:
; DBG-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0>
; DBG-NEXT: Successor(s): ir-bb<loop>
@@ -230,6 +230,9 @@ exit:
; DBG-NEXT: IR %for = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] (extra operand: vp<[[RESUME_P]]>)
; DBG: IR %ec = icmp slt i32 %iv.next.trunc, %n
; DBG-NEXT: No successors
+; DBG-EMPTY:
+; DBG-NEXT: ir-bb<exit>:
+; DBG-NEXT: No successors
; DBG-NEXT: }
define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index 68dd47537fdfdf..50755f8f05e35a 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -14,8 +14,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: compound=true
; CHECK-NEXT: N0 [label =
; CHECK-NEXT: "ir-bb\<for.body.preheader\>:\l" +
-; CHECK-NEXT: "No successors\l"
+; CHECK-NEXT: "Successor(s): vector.ph\l"
; CHECK-NEXT: ]
+; CHECK-NEXT: N0 -> N1 [ label=""]
; CHECK-NEXT: N1 [label =
; CHECK-NEXT: "vector.ph:\l" +
; CHECK-NEXT: "Successor(s): vector loop\l"
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 1f815899ed55c2..c342d2f81e979b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -9,6 +9,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -31,9 +34,6 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
; CHECK-EMPTY:
@@ -41,6 +41,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ]
; CHECK: IR %iv.next = add i64 %iv, 1
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
index 53f5a5658fb68d..f31ecca50a3701 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
@@ -3,15 +3,18 @@
define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; CHECK: VPlan 'Final VPlan for VF={2},UF={1}' {
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<2> = VF * UF
+; CHECK-NEXT: Live-in ir<%n.vec> = vector-trip-count
; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: IR %n.mod.vf = urem i64 %0, 2
+; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf
+; CHECK-NEXT: IR %ind.end = getelementptr i8, ptr %start, i64 %n.vec
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -76,25 +79,26 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): default.2
; CHECK-EMPTY:
; CHECK-NEXT: default.2:
-; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
+; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<2>
+; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<%n.vec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]>
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_CMP]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: ir-bb<scalar.ph>:
+; CHECK-NEXT: IR %bc.resume.val = phi ptr [ %ind.end, %middle.block ], [ %start, %entry ]
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop.header>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %bc.resume.val, %scalar.ph ], [ %ptr.iv.next, %loop.latch ]
; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
; CHECK-NEXT: No successors
; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index c9612ced3eee01..746e7f7059a6b9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -16,7 +16,7 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %and = and i64 %N, 15
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i4 (trunc i64 %N to i4) to i64)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -42,9 +42,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -53,20 +50,27 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]
; CHECK: IR %cmp = icmp eq i64 %iv.next, 0
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
; CHECK: Executing best plan with VF=8, UF=2
; CHECK-NEXT: VPlan 'Final VPlan for VF={8},UF={2}' {
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<16> = VF * UF
+; CHECK-NEXT: Live-in ir<%n.vec> = vector-trip-count
; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %and = and i64 %N, 15
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i4 (trunc i64 %N to i4) to i64)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: ir-bb<vector.ph>:
+; CHECK-NEXT: IR %n.mod.vf = urem i64 %and, 16
+; CHECK-NEXT: IR %n.vec = sub i64 %and, %n.mod.vf
+; CHECK-NEXT: IR %ind.end = sub i64 %and, %n.vec
+; CHECK-NEXT: IR %ind.end1 = getelementptr i8, ptr %A, i64 %n.vec
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -84,26 +88,28 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
; CHECK-NEXT: vp<[[VPTR4:%.+]]> = vector-pointer vp<[[PADD1]]>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VPTR3]]>, ir<%add>
; CHECK-NEXT: WIDEN store vp<[[VPTR4]]>, ir<%add>.1
-; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
+; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, ir<16>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
+; CHECK-NEXT: Successor(s): ir-bb<middle.block>
; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]>
+; CHECK-NEXT: ir-bb<middle.block>:
+; CHECK-NEXT: EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, ir<%n.vec>
; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: ir-bb<scalar.ph>:
+; CHECK-NEXT: IR %bc.resume.val = phi i64 [ %ind.end, %middle.block ], [ %and, %entry ]
+; CHECK-NEXT: IR %bc.resume.val2 = phi ptr [ %ind.end1, %middle.block ], [ %A, %entry ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ %and, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]
+; CHECK-NEXT: IR %iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %iv.next, %loop ]
+; CHECK-NEXT: IR %p.src = phi ptr [ %bc.resume.val2, %scalar.ph ], [ %p.src.next, %loop ]
; CHECK: IR %cmp = icmp eq i64 %iv.next, 0
; CHECK-NEXT: No successors
; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 50d406d0c04164..2adeb5920cb5b2 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -11,6 +11,9 @@ define void @foo(i64 %n) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<8> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -47,9 +50,6 @@ define void @foo(i64 %n) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<outer.header>
; CHECK-EMPTY:
@@ -59,6 +59,9 @@ define void @foo(i64 %n) {
; CHECK-NEXT: IR store i64 %outer.iv, ptr %gep.1, align 4
; CHECK-NEXT: IR %add = add nsw i64 %outer.iv, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
br label %outer.header
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 6bb20a301e0ade..8894b871320bcb 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -13,6 +13,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body.preheader>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -38,9 +41,6 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end.loopexit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end.loopexit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -48,6 +48,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end.loopexit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -77,6 +80,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body.preheader>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -105,9 +111,6 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end.loopexit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end.loopexit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -115,6 +118,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end.loopexit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -145,6 +151,9 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -170,10 +179,6 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]>)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
; CHECK-NEXT: Successor(s): ir-bb<for.body>
@@ -182,6 +187,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]>)
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -208,6 +217,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -233,9 +245,6 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
; CHECK-NEXT: Successor(s): ir-bb<for.body>
@@ -245,6 +254,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: IR %red = phi float [ %red.next, %for.body ], [ 0.000000e+00, %entry ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -275,7 +287,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 smax %n)
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -319,9 +331,6 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -329,6 +338,9 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
; CHECK-NEXT: IR %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
; CHECK-NEXT: IR %cmp = icmp ult i64 %i, 5
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -365,6 +377,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<256> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -396,9 +411,6 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -406,6 +418,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
; CHECK: IR %cmp = icmp slt i64 %iv.next, 1024
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -446,6 +461,9 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -475,10 +493,6 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>
-; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]>)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00>
; CHECK-NEXT: Successor(s): ir-bb<for.body>
@@ -488,6 +502,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: IR %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>
+; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]>)
+; CHECK-NEXT: No successors
; CHECK-NEXT:}
entry:
@@ -516,6 +534,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -566,9 +587,6 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -576,6 +594,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ]
; CHECK: IR %cmp1 = icmp slt i32 %lsd, 100
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT:}
;
entry:
@@ -624,7 +645,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: IR %inc = add i64 %div, 1
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
; CHECK-NEXT: EMIT vp<[[EXP_SCEV:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -651,9 +672,6 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<loop.exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop.exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -661,6 +679,9 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
; CHECK: IR %iv.next = add i64 %iv, %inc
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -691,6 +712,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -715,10 +739,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]>)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -726,6 +746,10 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %ec = icmp eq i32 %iv.next, 1000
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]>)
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -752,6 +776,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -779,9 +806,6 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -789,6 +813,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -818,6 +845,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -844,9 +874,6 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -854,6 +881,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -882,6 +912,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -929,9 +962,6 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<end>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -939,6 +969,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ]
; CHECK: IR %ifcond = fcmp oeq float %ld.value, 5.0
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<end>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -977,6 +1010,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -1003,9 +1039,6 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -1013,6 +1046,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -1041,6 +1077,9 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -1082,6 +1121,9 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1000> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -1110,10 +1152,6 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]>)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22>
; CHECK-NEXT: Successor(s): ir-bb<loop>
@@ -1123,6 +1161,10 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]>)
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index cdeffeff84d03a..a939b1e923a914 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -12,6 +12,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; CHECK-NEXT: ir<0> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -49,9 +52,6 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.end>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
@@ -60,6 +60,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: IR %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 0, %entry ]
; CHECK: IR %tmp5 = trunc i32 %tmp4 to i8
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.end>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 88e7aaccfe2f35..38bd54d8da9843 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -19,7 +19,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -88,7 +88,7 @@ exit:
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -173,7 +173,7 @@ exit:
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -258,6 +258,9 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<11> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
@@ -332,7 +335,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -430,7 +433,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -537,7 +540,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -644,7 +647,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -750,7 +753,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -816,7 +819,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -898,6 +901,9 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<999> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -962,6 +968,9 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<201> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -1033,6 +1042,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -1073,9 +1085,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
@@ -1089,6 +1098,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
; CHECK-NEXT: IR store i32 %l, ptr %gep.dst, align 16
; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, 0
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
@@ -1119,7 +1131,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %end to i64)) + (ptrtoint ptr %start to i64))
-; CHECK-NEXT: No successors
+; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
@@ -1164,9 +1176,6 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
; CHECK-EMPTY:
@@ -1176,6 +1185,9 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv.next, align 1
; CHECK-NEXT: IR %c.1 = icmp eq i8 %l, 0
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
index 27d81de260d3b9..9778ef6853a70f 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -13,6 +13,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) {
; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
; CHECK-NEXT: Live-in ir<%length> = original trip-count
; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
>From 382380aaa9e75b7f3e6819f025122902259d83ee Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 10 Nov 2024 10:05:07 +0000
Subject: [PATCH 09/10] !fixup formatting
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index eb31ff3623f918..960aa68e59d09c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1023,8 +1023,8 @@ void VPlan::execute(VPTransformState *State) {
replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh);
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
- LLVM_DEBUG(dbgs() << "Executing best plan with VF=" <<State->VF
- << ", UF=" << getUF()<< '\n');
+ LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF
+ << ", UF=" << getUF() << '\n');
setName("Final VPlan");
LLVM_DEBUG(dump());
>From e5b8af3a4c10eb64ce376336176ef00a666bf3ec Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 25 Nov 2024 21:16:49 +0000
Subject: [PATCH 10/10] !fixup address latest comments, thanks!
---
.../Transforms/Vectorize/LoopVectorize.cpp | 36 ++++++-------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 52 ++++++++++++-------
llvm/lib/Transforms/Vectorize/VPlan.h | 20 +++++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 5 +-
.../RISCV/vplan-vp-intrinsics-reduction.ll | 24 ++-------
.../Transforms/Vectorize/VPDomTreeTest.cpp | 9 ++--
.../Transforms/Vectorize/VPlanTest.cpp | 38 +++++---------
.../Vectorize/VPlanVerifierTest.cpp | 18 +++----
9 files changed, 99 insertions(+), 111 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f16dfdc4abf3b0..5c2ec637e9d9f1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2432,12 +2432,13 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
/// entry. This is used when adjusting \p Plan during skeleton
/// creation, i.e. adjusting the plan after introducing an initial runtime
/// check.
-static void connectScalarPreheaderInVPlan(VPlan &Plan) {
+/// TODO: Connect scalar preheader during initial VPlan construction.
+static void connectScalarPreheaderAsBypassInVPlan(VPlan &Plan) {
VPBlockBase *VectorPH = Plan.getVectorPreheader();
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
- VPBlockBase *PredVPB = Plan.getEntry();
- VPBlockUtils::connectBlocks(PredVPB, ScalarPH, -1, 0);
- VPBlockUtils::connectBlocks(PredVPB, VectorPH, 0, -1);
+ VPBlockBase *Entry = Plan.getEntry();
+ VPBlockUtils::connectBlocks(Entry, ScalarPH);
+ std::swap(Entry->getSuccessors()[0], Entry->getSuccessors()[1]);
}
/// Introduces a new VPIRBasicBlock for \p CheckIRBB to \p Plan between the
@@ -2447,6 +2448,7 @@ static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) {
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
VPBlockBase *VectorPH = Plan.getVectorPreheader();
VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor();
+ assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB);
VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH);
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB);
@@ -2534,7 +2536,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
LoopBypassBlocks.push_back(TCCheckBlock);
// TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here.
- connectScalarPreheaderInVPlan(Plan);
+ connectScalarPreheaderAsBypassInVPlan(Plan);
}
BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
@@ -7712,10 +7714,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// 0. Generate SCEV-dependent code into the preheader, including TripCount,
// before making any changes to the CFG.
- if (!BestVPlan.getEntry()->empty()) {
+ if (!BestVPlan.getPreheader()->empty()) {
State.CFG.PrevBB = OrigLoop->getLoopPreheader();
State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator());
- BestVPlan.getEntry()->execute(&State);
+ BestVPlan.getPreheader()->execute(&State);
}
if (!ILV.getTripCount())
ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0)));
@@ -7937,10 +7939,11 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
- VPBlockBase *VectorPH = Plan.getVectorPreheader();
- VPBlockBase *PredVPB = VectorPH->getSinglePredecessor();
- if (PredVPB->getNumSuccessors() == 1)
- connectScalarPreheaderInVPlan(Plan);
+ // Connect the vector preheader to the scalar preheader when creating the trip
+ // count for the epilogue loop. Otherwise there is already a runtime check and
+ // connection to the scalar preheader, so we introduce it as new check block.
+ if (ForEpilogue)
+ connectScalarPreheaderAsBypassInVPlan(Plan);
else
introduceCheckBlockInVPlan(Plan, TCCheckBlock);
return TCCheckBlock;
@@ -8093,14 +8096,9 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
VPBasicBlock *OldEntry = Plan.getEntry();
VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
Plan.setEntry(NewEntry);
- for (auto &R : make_early_inc_range(*NewEntry)) {
- auto *VPIR = dyn_cast<VPIRInstruction>(&R);
- if (!VPIR || !isa<PHINode>(VPIR->getInstruction()))
- break;
- VPIR->eraseFromParent();
- }
+ delete OldEntry;
- connectScalarPreheaderInVPlan(Plan);
+ connectScalarPreheaderAsBypassInVPlan(Plan);
return Insert;
}
@@ -10317,7 +10315,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// should be removed once induction resume value creation is done
// directly in VPlan.
EpilogILV.setTripCount(MainILV.getTripCount());
- for (auto &R : make_early_inc_range(*BestEpiPlan.getEntry())) {
+ for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpandR)
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a8a5e7c765a5f9..75ca717b7c1434 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -170,8 +170,7 @@ VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
}
void VPBlockBase::setPlan(VPlan *ParentPlan) {
- assert(ParentPlan->getEntry() == this &&
- "Can only set plan on its entry or preheader block.");
+ assert(ParentPlan->getEntry() == this && "Can only set plan on its entry.");
Plan = ParentPlan;
}
@@ -822,6 +821,18 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
}
#endif
+VPlan::VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry,
+ VPIRBasicBlock *ScalarHeader)
+ : VPlan(Preheader, TC, ScalarHeader) {
+ VPBlockUtils::connectBlocks(Preheader, Entry);
+}
+
+VPlan::VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry,
+ VPIRBasicBlock *ScalarHeader)
+ : VPlan(Preheader, ScalarHeader) {
+ VPBlockUtils::connectBlocks(Preheader, Entry);
+}
+
VPlan::~VPlan() {
if (Entry) {
VPValue DummyValue;
@@ -851,6 +862,9 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
VPIRBasicBlock *Entry =
VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader());
VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph");
+ // Connect entry only to vector preheader initially. Edges to the scalar
+ // preheader will be inserted later, during skeleton creation when runtime
+ // guards are added as needed.
VPBlockUtils::connectBlocks(Entry, VecPreheader);
VPIRBasicBlock *ScalarHeader =
VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader());
@@ -997,20 +1011,21 @@ void VPlan::execute(VPTransformState *State) {
BasicBlock *VectorPreHeader = State->CFG.PrevBB;
State->Builder.SetInsertPoint(VectorPreHeader->getTerminator());
- replaceVPBBWithIRVPBB(
- cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor()),
- VectorPreHeader);
+ // Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
+ cast<BranchInst>(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr);
State->CFG.DTU.applyUpdates(
{{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}});
- // Replace regular VPBB's for the middle and scalar preheader blocks with
- // VPIRBasicBlocks wrapping their IR blocks. The IR blocks are created during
- // skeleton creation, so we can only create the VPIRBasicBlocks now during
- // VPlan execution rather than earlier during VPlan construction.
+ // Replace regular VPBB's for the vector preheader, middle and scalar
+ // preheader blocks with VPIRBasicBlocks wrapping their IR blocks. The IR
+ // blocks are created during skeleton creation, so we can only create the
+ // VPIRBasicBlocks now during VPlan execution rather than earlier during VPlan
+ // construction.
+ replaceVPBBWithIRVPBB(getVectorPreheader(), VectorPreHeader);
BasicBlock *MiddleBB = State->CFG.ExitBB;
- VPBasicBlock *MiddleVPBB = getMiddleBlock();
BasicBlock *ScalarPh = MiddleBB->getSingleSuccessor();
replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh);
+ VPBasicBlock *MiddleVPBB = getMiddleBlock();
replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB);
LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF
@@ -1033,8 +1048,9 @@ void VPlan::execute(VPTransformState *State) {
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Entry);
- // Generate code in the loop pre-header and body.
- for (VPBlockBase *Block : make_range(RPOT.begin(), RPOT.end()))
+ // Generate code for the VPlan, in parts of the vector skeleton, loop body and
+ // successor blocks including the middle, exit and scalar preheader blocks.
+ for (VPBlockBase *Block : RPOT)
Block->execute(State);
VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
@@ -1280,14 +1296,10 @@ VPlan *VPlan::duplicate() {
VPBasicBlock *VPlan::getScalarPreheader() {
auto *MiddleVPBB =
cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
- if (MiddleVPBB->getNumSuccessors() == 2) {
- // Order is strict: first is the exit block, second is the scalar preheader.
- return cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[1]);
- }
- if (auto *IRVPBB = dyn_cast<VPBasicBlock>(MiddleVPBB->getSingleSuccessor()))
- return IRVPBB;
-
- return nullptr;
+ auto *LastSucc = MiddleVPBB->getSuccessors().back();
+ // Order is strict: if the last successor is VPIRBasicBlock, it must be the
+ // single exit.
+ return isa<VPIRBasicBlock>(LastSucc) ? nullptr : cast<VPBasicBlock>(LastSucc);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8418b42cfec754..26cd0b549bd97c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3715,8 +3715,9 @@ class VPlan {
friend class VPlanPrinter;
friend class VPSlotTracker;
- /// Hold the single entry to the Hierarchical CFG of the VPlan, i.e. the
- /// preheader of the vector loop.
+ /// VPBasicBlock corresponding to the original preheader. Used to place
+ /// VPExpandSCEV recipes for expressions used during skeleton creation and the
+ /// rest of VPlan execution.
VPBasicBlock *Entry;
/// VPIRBasicBlock wrapping the header of the original scalar loop.
@@ -3763,12 +3764,21 @@ class VPlan {
DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
public:
- /// Construct a VPlan with \p Entry entering the plan and trip count \p TC.
+ /// Construct a VPlan with \p Entry entering the plan, trip count \p TC and
+ /// with \p ScalarHeader wrapping the original header of the scalar loop.
VPlan(VPBasicBlock *Entry, VPValue *TC, VPIRBasicBlock *ScalarHeader)
: VPlan(Entry, ScalarHeader) {
TripCount = TC;
}
+ /// Constructor variants that take disconnected preheader and entry blocks,
+ /// connecting them as part of construction. Only used to reduce the need of
+ /// code changes during transition.
+ VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry,
+ VPIRBasicBlock *ScalarHeader);
+ VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry,
+ VPIRBasicBlock *ScalarHeader);
+
/// Construct a VPlan with \p Entry to the plan.
VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
: Entry(Entry), ScalarHeader(ScalarHeader) {
@@ -3968,6 +3978,10 @@ class VPlan {
SCEVToExpansion[S] = V;
}
+ /// \return The block corresponding to the original preheader.
+ VPBasicBlock *getPreheader() { return Entry; }
+ const VPBasicBlock *getPreheader() const { return Entry; }
+
/// Clone the current VPlan, update all VPValues of the new VPlan and cloned
/// recipes to refer to the clones, and return it.
VPlan *duplicate();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cb1f9b4bf346fb..54f3bbde3b9c94 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3241,7 +3241,13 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
void VPExpandSCEVRecipe::execute(VPTransformState &State) {
assert(!State.Lane && "cannot be used in per-lane");
if (State.ExpandedSCEVs.contains(Expr)) {
+ // SCEV Expr has already been expanded, result must already be set. At the
+ // moment we have to execute the entry block twice (once before skeleton
+ // creation to get expanded SCEVs used by the skeleton and once during
+ // regular VPlan execution).
State.Builder.SetInsertPoint(State.CFG.VPBB2IRBB[getParent()]);
+ assert(State.get(this, VPLane(0)) == State.ExpandedSCEVs[Expr] &&
+ "Results must match");
return;
}
@@ -3249,8 +3255,6 @@ void VPExpandSCEVRecipe::execute(VPTransformState &State) {
SCEVExpander Exp(SE, DL, "induction");
Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
&*State.Builder.GetInsertPoint());
- /* assert(!State.ExpandedSCEVs.contains(Expr) &&*/
- /*"Same SCEV expanded multiple times");*/
State.ExpandedSCEVs[Expr] = Res;
State.set(this, Res, VPLane(0));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 33ebff1dee312f..2a1ac4d96201e1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -377,9 +377,8 @@ static bool mergeBlocksIntoPredecessors(VPlan &Plan) {
continue;
auto *PredVPBB =
dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
- if (!PredVPBB || PredVPBB->getNumSuccessors() != 1)
- continue;
- if (isa<VPIRBasicBlock>(PredVPBB))
+ if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
+ isa<VPIRBasicBlock>(PredVPBB))
continue;
WorkList.push_back(VPBB);
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 199d6a7c858bc5..e7eb5778ffb930 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -62,10 +62,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-cond ir<true>
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; IF-EVL-OUTLOOP-EMPTY:
-; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; IF-EVL-OUTLOOP-NEXT: No successors
-; IF-EVL-OUTLOOP-EMPTY:
; IF-EVL-OUTLOOP-NEXT: scalar.ph:
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -77,7 +73,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: No successors
; IF-EVL-OUTLOOP-EMPTY:
; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
; IF-EVL-OUTLOOP-NEXT: No successors
; IF-EVL-OUTLOOP-NEXT: }
;
@@ -116,10 +112,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: EMIT branch-on-cond ir<true>
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; IF-EVL-INLOOP-EMPTY:
-; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; IF-EVL-INLOOP-NEXT: No successors
-; IF-EVL-INLOOP-EMPTY:
; IF-EVL-INLOOP-NEXT: scalar.ph:
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -131,7 +123,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: No successors
; IF-EVL-INLOOP-EMPTY:
; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
; IF-EVL-INLOOP-NEXT: No successors
; IF-EVL-INLOOP-NEXT: }
;
@@ -166,10 +158,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; NO-VP-OUTLOOP-EMPTY:
-; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; NO-VP-OUTLOOP-NEXT: No successors
-; NO-VP-OUTLOOP-EMPTY:
; NO-VP-OUTLOOP-NEXT: scalar.ph:
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -181,7 +169,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-OUTLOOP-NEXT: No successors
; NO-VP-OUTLOOP-EMPTY:
; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
; NO-VP-OUTLOOP-NEXT: No successors
; NO-VP-OUTLOOP-NEXT: }
;
@@ -216,10 +204,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
; NO-VP-INLOOP-EMPTY:
-; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
-; NO-VP-INLOOP-NEXT: No successors
-; NO-VP-INLOOP-EMPTY:
; NO-VP-INLOOP-NEXT: scalar.ph:
; NO-VP-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start>
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.body>
@@ -231,7 +215,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-INLOOP-NEXT: No successors
; NO-VP-INLOOP-EMPTY:
; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>)
+; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
; NO-VP-INLOOP-NEXT: No successors
; NO-VP-INLOOP-NEXT: }
;
diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
index 604f87b634bba6..86182ccae0b55b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
@@ -44,9 +44,8 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
@@ -125,9 +124,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
@@ -208,9 +206,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
VPDominatorTree VPDT;
VPDT.recalculate(Plan);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 13357738eea6d2..3179cfc676ab67 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -259,9 +259,8 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, VPBB2->getPlan());
@@ -282,9 +281,8 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, R1->getPlan());
@@ -315,9 +313,8 @@ TEST(VPBasicBlockTest, getPlan) {
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
EXPECT_EQ(&Plan, VPBB1->getPlan());
EXPECT_EQ(&Plan, R1->getPlan());
@@ -362,9 +359,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
}
{
@@ -465,9 +461,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB0);
VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB);
}
{
@@ -551,9 +546,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
}
{
@@ -602,9 +596,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
}
{
@@ -697,9 +690,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
// Use Plan to properly clean up created blocks.
auto TC = std::make_unique<VPValue>();
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
}
delete ScalarHeader;
}
@@ -741,9 +733,8 @@ TEST(VPBasicBlockTest, print) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "scalar.header");
auto * ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPBB0, VPBB1);
VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, TC, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB);
std::string FullDump;
raw_string_ostream OS(FullDump);
Plan.printDOT(OS);
@@ -834,9 +825,8 @@ TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPBB0, VPBB1);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, TC, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB);
Plan.setName("TestPlan");
Plan.addVF(ElementCount::getFixed(4));
@@ -1306,13 +1296,11 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
TEST(VPRecipeTest, dumpRecipeInPlan) {
VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
VPBasicBlock *VPBB1 = new VPBasicBlock();
- VPBlockUtils::connectBlocks(VPBB0, VPBB1);
-
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB);
IntegerType *Int32 = IntegerType::get(C, 32);
auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32),
@@ -1380,13 +1368,11 @@ TEST(VPRecipeTest, dumpRecipeInPlan) {
TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
VPBasicBlock *VPBB1 = new VPBasicBlock();
- VPBlockUtils::connectBlocks(VPBB0, VPBB1);
-
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB);
- VPlan Plan(VPBB0, ScalarHeaderVPBB);
+ VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB);
IntegerType *Int32 = IntegerType::get(C, 32);
auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32),
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 2c7566c973c4e1..edb3f8a2952942 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -32,9 +32,8 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -70,9 +69,8 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -118,9 +116,8 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
auto TC = std::make_unique<VPValue>();
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -160,9 +157,8 @@ TEST(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -203,9 +199,8 @@ TEST(VPVerifierTest, DuplicateSuccessorsInsideRegion) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, VPBB1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -238,9 +233,8 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) {
LLVMContext C;
auto *ScalarHeader = BasicBlock::Create(C, "");
VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader);
- VPBlockUtils::connectBlocks(VPPH, R1);
VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB);
- VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB);
+ VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
More information about the llvm-commits
mailing list