[llvm] 34c2dcd - [VPlan] Move initial skeleton construction to createInitialVPlan. (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 12 05:01:35 PST 2023
Author: Florian Hahn
Date: 2023-11-12T13:00:44Z
New Revision: 34c2dcd5ac26fe4e8313091131341de7945441ea
URL: https://github.com/llvm/llvm-project/commit/34c2dcd5ac26fe4e8313091131341de7945441ea
DIFF: https://github.com/llvm/llvm-project/commit/34c2dcd5ac26fe4e8313091131341de7945441ea.diff
LOG: [VPlan] Move initial skeleton construction to createInitialVPlan. (NFC)
This patch moves creating the middle VPBBs and an initial empty
vector loop region for the top-level loop to createInitialVPlan.
This consolidates code to create the initial VPlan skeleton and enables
adding other bits outside the main region during initial VPlan
construction. In particular, D150398 will add the exit check & branch to
the middle block.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D158333
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2156e66a8071042..65f0662320de8a5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8777,14 +8777,13 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
VPBasicBlock *HeaderVPBB = new VPBasicBlock("vector.body");
VPBasicBlock *LatchVPBB = new VPBasicBlock("vector.latch");
VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB);
- auto *TopRegion = new VPRegionBlock(HeaderVPBB, LatchVPBB, "vector loop");
- VPBlockUtils::insertBlockAfter(TopRegion, Plan->getEntry());
- VPBasicBlock *MiddleVPBB = new VPBasicBlock("middle.block");
- VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
+ Plan->getVectorLoopRegion()->setEntry(HeaderVPBB);
+ Plan->getVectorLoopRegion()->setExiting(LatchVPBB);
// Don't use getDecisionAndClampRange here, because we don't know the UF
// so this function is better to be conservative, rather than to split
// it up into
diff erent VPlans.
+ // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
bool IVUpdateMayOverflow = false;
for (ElementCount VF : Range)
IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8074a2e83f14ba3..9ffce475f022eb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -714,6 +714,11 @@ VPlanPtr VPlan::createInitialVPlan(const SCEV *TripCount, ScalarEvolution &SE) {
auto Plan = std::make_unique<VPlan>(Preheader, VecPreheader);
Plan->TripCount =
vputils::getOrCreateVPValueForSCEVExpr(*Plan, TripCount, SE);
+ // Create empty VPRegionBlock, to be filled during processing later.
+ auto *TopRegion = new VPRegionBlock("vector loop", false /*isReplicator*/);
+ VPBlockUtils::insertBlockAfter(TopRegion, VecPreheader);
+ VPBasicBlock *MiddleVPBB = new VPBasicBlock("middle.block");
+ VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
return Plan;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 96cfc9354ead806..a26308a212bbd3c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2561,8 +2561,11 @@ class VPlan {
~VPlan();
- /// Create an initial VPlan with preheader and entry blocks. Creates a
- /// VPExpandSCEVRecipe for \p TripCount and uses it as plan's trip count.
+ /// Create initial VPlan skeleton, having an "entry" VPBasicBlock (wrapping
+ /// original scalar pre-header) which contains SCEV expansions that need to
+ /// happen before the CFG is modified; a VPBasicBlock for the vector
+ /// pre-header, followed by a region for the vector loop, followed by the
+ /// middle VPBasicBlock.
static VPlanPtr createInitialVPlan(const SCEV *TripCount,
ScalarEvolution &PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index a49332f39ceebef..e90f5b94a64238b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -167,8 +167,9 @@ VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
}
// Create new VPBB.
- LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << BB->getName() << "\n");
- VPBasicBlock *VPBB = new VPBasicBlock(BB->getName());
+ StringRef Name = isHeaderBB(BB, TheLoop) ? "vector.body" : BB->getName();
+ LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
+ VPBasicBlock *VPBB = new VPBasicBlock(Name);
BB2VPBB[BB] = VPBB;
// Get or create a region for the loop containing BB.
@@ -176,22 +177,25 @@ VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
if (!LoopOfBB)
return VPBB;
- VPRegionBlock *RegionOfBB = Loop2Region.lookup(LoopOfBB);
- assert((RegionOfBB != nullptr) ^ isHeaderBB(BB, LoopOfBB) &&
- "region must exist or BB must be a loop header");
- if (RegionOfBB) {
- VPBB->setParent(RegionOfBB);
+ auto *RegionOfVPBB = Loop2Region.lookup(LoopOfBB);
+ if (!isHeaderBB(BB, LoopOfBB)) {
+ assert(RegionOfVPBB &&
+ "Region should have been created by visiting header earlier");
+ VPBB->setParent(RegionOfVPBB);
+ return VPBB;
+ }
+
+ assert(!RegionOfVPBB &&
+ "First visit of a header basic block expects to register its region.");
+ // Handle a header - take care of its Region.
+ if (LoopOfBB == TheLoop) {
+ RegionOfVPBB = Plan.getVectorLoopRegion();
} else {
- // If BB's loop is nested inside another loop within VPlan's scope, the
- // header of that enclosing loop was already visited and its region
- // constructed and recorded in Loop2Region. That region is now set as the
- // parent of VPBB's region. Otherwise it is set to null.
- auto *RegionOfVPBB = new VPRegionBlock(
- LoopOfBB->getHeader()->getName().str(), false /*isReplicator*/);
+ RegionOfVPBB = new VPRegionBlock(Name.str(), false /*isReplicator*/);
RegionOfVPBB->setParent(Loop2Region[LoopOfBB->getParentLoop()]);
- RegionOfVPBB->setEntry(VPBB);
- Loop2Region[LoopOfBB] = RegionOfVPBB;
}
+ RegionOfVPBB->setEntry(VPBB);
+ Loop2Region[LoopOfBB] = RegionOfVPBB;
return VPBB;
}
@@ -314,6 +318,25 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// Main interface to build the plain CFG.
void PlainCFGBuilder::buildPlainCFG() {
+ // 0. Reuse the top-level region, vector-preheader and exit VPBBs from the
+ // skeleton. These were created directly rather than via getOrCreateVPBB(),
+ // revisit them now to update BB2VPBB. Note that header/entry and
+ // latch/exiting VPBB's of top-level region have yet to be created.
+ VPRegionBlock *TheRegion = Plan.getVectorLoopRegion();
+ BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
+ assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+ "Unexpected loop preheader");
+ auto *VectorPreheaderVPBB =
+ cast<VPBasicBlock>(TheRegion->getSinglePredecessor());
+ // ThePreheaderBB conceptually corresponds to both Plan.getPreheader() (which
+ // wraps the original preheader BB) and Plan.getEntry() (which represents the
+ // new vector preheader); here we're interested in setting BB2VPBB to the
+ // latter.
+ BB2VPBB[ThePreheaderBB] = VectorPreheaderVPBB;
+ BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
+ assert(LoopExitBB && "Loops with multiple exits are not supported.");
+ BB2VPBB[LoopExitBB] = cast<VPBasicBlock>(TheRegion->getSingleSuccessor());
+
// 1. Scan the body of the loop in a topological order to visit each basic
// block after having visited its predecessor basic blocks. Create a VPBB for
// each BB and link it to its successor and predecessor VPBBs. Note that
@@ -323,25 +346,11 @@ void PlainCFGBuilder::buildPlainCFG() {
// Loop PH needs to be explicitly visited since it's not taken into account by
// LoopBlocksDFS.
- BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
- assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
- "Unexpected loop preheader");
- // buildPlainCFG needs to be called after createInitialVPlan, which creates
- // the initial skeleton (including the preheader VPBB). buildPlainCFG builds
- // the CFG for the loop nest and hooks it up to the initial skeleton.
- VPBasicBlock *ThePreheaderVPBB = Plan.getEntry();
- BB2VPBB[ThePreheaderBB] = ThePreheaderVPBB;
- ThePreheaderVPBB->setName("vector.ph");
for (auto &I : *ThePreheaderBB) {
if (I.getType()->isVoidTy())
continue;
IRDef2VPValue[&I] = Plan.getVPValueOrAddLiveIn(&I);
}
- // Create region (and header block) for the outer loop, so that we can link
- // PH->Region.
- VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader());
- HeaderVPBB->setName("vector.body");
- ThePreheaderVPBB->setOneSuccessor(HeaderVPBB->getParent());
LoopBlocksRPO RPO(TheLoop);
RPO.perform(LI);
@@ -354,12 +363,14 @@ void PlainCFGBuilder::buildPlainCFG() {
createVPInstructionsForVPBB(VPBB, BB);
Loop *LoopForBB = LI->getLoopFor(BB);
// Set VPBB predecessors in the same order as they are in the incoming BB.
- if (!isHeaderBB(BB, LoopForBB))
+ if (!isHeaderBB(BB, LoopForBB)) {
setVPBBPredsFromBB(VPBB, BB);
- else {
- // BB is a loop header, set the predecessor for the region.
+ } else {
+ // BB is a loop header, set the predecessor for the region, except for the
+ // top region, whose predecessor was set when creating VPlan's skeleton.
assert(isHeaderVPBB(VPBB) && "isHeaderBB and isHeaderVPBB disagree");
- setRegionPredsFromBB(Region, BB);
+ if (TheRegion != Region)
+ setRegionPredsFromBB(Region, BB);
}
// Set VPBB successors. We create empty VPBBs for successors if they don't
@@ -387,22 +398,16 @@ void PlainCFGBuilder::buildPlainCFG() {
continue;
}
// For a latch we need to set the successor of the region rather than that
- // of VPBB and it should be set to the exit, i.e., non-header successor.
- Region->setOneSuccessor(isHeaderVPBB(Successor0) ? Successor1 : Successor0);
+ // of VPBB and it should be set to the exit, i.e., non-header successor,
+ // except for the top region, whose successor was set when creating VPlan's
+ // skeleton.
+ if (TheRegion != Region)
+ Region->setOneSuccessor(isHeaderVPBB(Successor0) ? Successor1
+ : Successor0);
Region->setExiting(VPBB);
}
- // 2. Process outermost loop exit. We created an empty VPBB for the loop
- // single exit BB during the RPO traversal of the loop body but Instructions
- // weren't visited because it's not part of the loop.
- BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
- assert(LoopExitBB && "Loops with multiple exits are not supported.");
- VPBasicBlock *LoopExitVPBB = BB2VPBB[LoopExitBB];
- // Loop exit was already set as successor of the loop exiting BB.
- // We only set its predecessor VPBB now.
- setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB);
-
- // 3. The whole CFG has been built at this point so all the input Values must
+ // 2. The whole CFG has been built at this point so all the input Values must
// have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding
// VPlan operands.
fixPhiNodes();
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index f9e717e62f0b23f..65df5973ac75b6b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -11,9 +11,9 @@ define void @foo(i64 %n) {
; CHECK-NEXT: ir<8> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): outer.header
+; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
-; CHECK-NEXT: <x1> outer.header: {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
@@ -39,9 +39,9 @@ define void @foo(i64 %n) {
; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): exit
+; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
-; CHECK-NEXT: exit:
+; CHECK-NEXT: middle.block:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 5381fa461e9c992..3f6a72e927e1572 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -112,12 +112,12 @@ compound=true
]
N1 [label =
"vector.ph:\l" +
- "Successor(s): for.body\l"
+ "Successor(s): vector loop\l"
]
N1 -> N2 [ label="" lhead=cluster_N3]
subgraph cluster_N3 {
fontname=Courier
- label="\<x1\> for.body"
+ label="\<x1\> vector loop"
N2 [label =
"vector.body:\l" +
" WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" +
@@ -133,7 +133,7 @@ compound=true
}
N2 -> N4 [ label="" ltail=cluster_N3]
N4 [label =
- "for.end:\l" +
+ "middle.block:\l" +
"No successors\l"
]
}
More information about the llvm-commits
mailing list