[llvm] 6383a12 - [VPlan] Refactor HCFG builder to preserve original vector latch (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 25 05:32:21 PST 2025
Author: Florian Hahn
Date: 2025-01-25T13:32:01Z
New Revision: 6383a12e3b4339fa4743bb97da4d51dea6d2e2ea
URL: https://github.com/llvm/llvm-project/commit/6383a12e3b4339fa4743bb97da4d51dea6d2e2ea
DIFF: https://github.com/llvm/llvm-project/commit/6383a12e3b4339fa4743bb97da4d51dea6d2e2ea.diff
LOG: [VPlan] Refactor HCFG builder to preserve original vector latch (NFC).
Update HCFG builder to preserve the original latch block of the initial
VPlan, ensuring there is always a latch.
It also skips creating the BranchOnCond for the latch of the top-level
loop, instead of removing it later. Exiting via the latch is controlled
by later recipes.
This further unifies HCFG construction and prepares for use to also
build an initial VPlan (VPlan0) for inner loops.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49694eb68e25bc..3a4f637f177e19 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9509,12 +9509,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
[this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
*PSE.getSE(), *TLI);
- // Remove the existing terminator of the exiting block of the top-most region.
- // A BranchOnCount will be added instead when adding the canonical IV recipes.
- auto *Term =
- Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
- Term->eraseFromParent();
-
// Tail folding is not supported for outer loops, so the induction increment
// is guaranteed to not wrap.
bool HasNUW = true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 4159a71469bd12..83c54a9b9c259c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1628,6 +1628,9 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
VFRange SubRange = {VF, MaxVFTimes2};
auto Plan = buildVPlan(SubRange);
VPlanTransforms::optimize(*Plan);
+ // Update the name of the latch of the top-level vector loop region region
+ // after optimizations which includes block folding.
+ Plan->getVectorLoopRegion()->getExiting()->setName("vector.latch");
VPlans.push_back(std::move(Plan));
VF = SubRange.End;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 0f3aa8d08e7b81..32723e5db9c457 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -292,6 +292,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
"Instruction shouldn't have been visited.");
if (auto *Br = dyn_cast<BranchInst>(Inst)) {
+ if (TheLoop->getLoopLatch() == BB ||
+ any_of(successors(BB),
+ [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
+ continue;
+
// Conditional branch instruction are represented using BranchOnCond
// recipes.
if (Br->isConditional()) {
@@ -356,11 +361,6 @@ void PlainCFGBuilder::buildPlainCFG() {
VPBasicBlock *VectorLatchVPBB = TheRegion->getExitingBasicBlock();
BB2VPBB[TheLoop->getHeader()] = VectorHeaderVPBB;
VectorHeaderVPBB->clearSuccessors();
- VectorLatchVPBB->clearPredecessors();
- if (TheLoop->getHeader() != TheLoop->getLoopLatch())
- BB2VPBB[TheLoop->getLoopLatch()] = VectorLatchVPBB;
- else
- TheRegion->setExiting(VectorHeaderVPBB);
// 1. Scan the body of the loop in a topological order to visit each basic
// block after having visited its predecessor basic blocks. Create a VPBB for
@@ -398,6 +398,13 @@ void PlainCFGBuilder::buildPlainCFG() {
setRegionPredsFromBB(Region, BB);
}
+ if (TheLoop->getLoopLatch() == BB) {
+ VPBB->setOneSuccessor(VectorLatchVPBB);
+ VectorLatchVPBB->clearPredecessors();
+ VectorLatchVPBB->setPredecessors({VPBB});
+ continue;
+ }
+
// Set VPBB successors. We create empty VPBBs for successors if they don't
// exist already. Recipes will be created when the successor is visited
// during the RPO traversal.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 2adeb5920cb5b2..52b2bcd9aac111 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -35,12 +35,14 @@ define void @foo(i64 %n) {
; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
-; CHECK-NEXT: vector.latch:
+; CHECK-NEXT: outer.latch:
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
+; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.latch:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 19c2483d34ed17..dcdaf008e10fe4 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -48,16 +48,19 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
EXPECT_EQ(0u, Entry->getNumPredecessors());
EXPECT_EQ(1u, Entry->getNumSuccessors());
- // Check that the region following the preheader is a single basic-block
- // region (loop).
+ // Check that the region following the preheader consists of a block for the
+ // original header and a separate latch.
VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
- EXPECT_EQ(8u, VecBB->size());
+ EXPECT_EQ(7u, VecBB->size());
EXPECT_EQ(0u, VecBB->getNumPredecessors());
- EXPECT_EQ(0u, VecBB->getNumSuccessors());
+ EXPECT_EQ(1u, VecBB->getNumSuccessors());
EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
- EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
EXPECT_EQ(&*Plan, VecBB->getPlan());
+ VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+ EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+ EXPECT_EQ(0u, VecLatch->getNumSuccessors());
+
auto Iter = VecBB->begin();
VPWidenPHIRecipe *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
EXPECT_NE(nullptr, Phi);
@@ -127,29 +130,33 @@ compound=true
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" +
" EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" +
- " EMIT branch-on-cond ir\<%exitcond\>\l" +
+ "Successor(s): vector.latch\l"
+ ]
+ N2 -> N4 [ label=""]
+ N4 [label =
+ "vector.latch:\l" +
"No successors\l"
]
}
- N2 -> N4 [ label="" ltail=cluster_N3]
- N4 [label =
+ N4 -> N5 [ label="" ltail=cluster_N3]
+ N5 [label =
"middle.block:\l" +
" EMIT vp\<%cmp.n\> = icmp eq ir\<%N\>, vp\<%0\>\l" +
" EMIT branch-on-cond vp\<%cmp.n\>\l" +
"Successor(s): ir-bb\<for.end\>, scalar.ph\l"
]
- N4 -> N5 [ label="T"]
- N4 -> N6 [ label="F"]
- N5 [label =
+ N5 -> N6 [ label="T"]
+ N5 -> N7 [ label="F"]
+ N6 [label =
"ir-bb\<for.end\>:\l" +
"No successors\l"
]
- N6 [label =
+ N7 [label =
"scalar.ph:\l" +
"Successor(s): ir-bb\<for.body\>\l"
]
- N6 -> N7 [ label=""]
- N7 [label =
+ N7 -> N8 [ label=""]
+ N8 [label =
"ir-bb\<for.body\>:\l" +
" IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" +
" IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" +
@@ -204,14 +211,17 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
EXPECT_EQ(0u, Entry->getNumPredecessors());
EXPECT_EQ(1u, Entry->getNumSuccessors());
- // Check that the region following the preheader is a single basic-block
- // region (loop).
+ // Check that the region following the preheader consists of a block for the
+ // original header and a separate latch.
VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
- EXPECT_EQ(8u, VecBB->size());
+ EXPECT_EQ(7u, VecBB->size());
EXPECT_EQ(0u, VecBB->getNumPredecessors());
- EXPECT_EQ(0u, VecBB->getNumSuccessors());
+ EXPECT_EQ(1u, VecBB->getNumSuccessors());
EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
- EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
+
+ VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+ EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+ EXPECT_EQ(0u, VecLatch->getNumSuccessors());
auto Iter = VecBB->begin();
EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
@@ -221,7 +231,6 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryRecipe>(&*Iter++));
EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
- EXPECT_NE(nullptr, dyn_cast<VPInstruction>(&*Iter++));
EXPECT_EQ(VecBB->end(), Iter);
}
More information about the llvm-commits
mailing list