[llvm] [VPlan] Convert more VPlan transforms to use RUN_VPLAN_PASS (PR #205007)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 12:05:49 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/205007
>From 07284c165037d7195c5747913f6a579a1f987f8b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 8 Jun 2026 17:43:42 +0200
Subject: [PATCH] [VPlan] Only check header-phi placement while in region form.
A VPHeaderPHIRecipe must reside in a header block. After the loop regions
are dissolved, removeBranchOnConst removes the backedge of
single-iteration loops, after which the former header block is no longer
recognized as a header (it loses its second predecessor) while its header
phi recipes remain until simplifyRecipes folds the now single-entry phis
away. Restrict the check to plans that still have a vector loop region.
With the header-phi placement check and the earlier CurrentIteration and
VPReductionPHIRecipe::getNumIncoming fixes in place, the remaining VPlan
transforms in executePlan after dissolveLoopRegions can use RUN_VPLAN_PASS.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 ++++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 ++++++--
.../VPlan/vplan-print-before-after-all.ll | 11 ++++++
3 files changed, 43 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3dbee08e7d7d8..1db7b902b5153 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5946,35 +5946,39 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// Remove dead back-edges for single-iteration loops with BranchOnCond(true).
// Only process loop latches to avoid removing edges from the middle block,
// which may be needed for epilogue vectorization.
- VPlanTransforms::removeBranchOnConst(BestVPlan, /*OnlyLatches=*/true);
- VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
+ RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
+ /*OnlyLatches=*/true);
+ RUN_VPLAN_PASS(VPlanTransforms::materializeBackedgeTakenCount, BestVPlan,
+ VectorPH);
std::optional<uint64_t> MaxRuntimeStep;
if (auto MaxVScale = getMaxVScale(*CM.TheFunction, CM.TTI))
MaxRuntimeStep = uint64_t(*MaxVScale) * BestVF.getKnownMinValue() * BestUF;
- VPlanTransforms::materializeVectorTripCount(
- BestVPlan, VectorPH, CM.foldTailByMasking(),
- CM.requiresScalarEpilogue(BestVF.isVector()), &BestVPlan.getVFxUF(),
- MaxRuntimeStep);
- VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
+ RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
+ VectorPH, CM.foldTailByMasking(),
+ CM.requiresScalarEpilogue(BestVF.isVector()),
+ &BestVPlan.getVFxUF(), MaxRuntimeStep);
+ RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
+ BestVF);
// Limit expansions to VPInstruction to when not vectorizing the epilogue.
// Currently this code path still relies on code re-using SCEVs expanded
// directly to IR instructions.
if (EpilogueVecKind == EpilogueVectorizationKind::None)
- VPlanTransforms::expandSCEVsToVPInstructions(BestVPlan, *PSE.getSE());
- VPlanTransforms::cse(BestVPlan);
- VPlanTransforms::simplifyRecipes(BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
+ *PSE.getSE());
+ RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
// Removing branches and incoming values may expose additional simplification
// opportunities.
- if (VPlanTransforms::removeBranchOnConst(BestVPlan,
- /*OnlyLatches=*/EpilogueVecKind !=
- EpilogueVectorizationKind::None))
- VPlanTransforms::simplifyRecipes(BestVPlan);
- VPlanTransforms::simplifyKnownEVL(BestVPlan, BestVF, PSE);
+ if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
+ /*OnlyLatches=*/EpilogueVecKind !=
+ EpilogueVectorizationKind::None))
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
// 0. Generate SCEV-dependent code in the entry, including TripCount, before
// making any changes to the CFG.
DenseMap<const SCEV *, Value *> ExpandedSCEVs =
- VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
+ RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
@@ -5990,7 +5994,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
replaceVPBBWithIRVPBB(ScalarPH, State.CFG.PrevBB->getSingleSuccessor(),
&BestVPlan);
- VPlanTransforms::removeDeadRecipes(BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d1899db20b20e..9c42897665591 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2808,11 +2808,18 @@ bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
"There must be a single edge between VPBB and its successor");
// Values coming from VPBB into phi recipes of RemovedSucc are removed from
- // these recipes.
- auto Phis = RemovedSucc->phis();
- for (VPRecipeBase &R : Phis)
+ // these recipes and single-entry header phis are removed.
+ for (VPRecipeBase &R : make_early_inc_range(RemovedSucc->phis())) {
cast<VPPhiAccessors>(&R)->removeIncomingValueFor(VPBB);
- SimplifiedPhi |= !std::empty(Phis);
+ SimplifiedPhi = true;
+ // Remove now invalid header phis that are left single-entry after
+ // removing their backedges.
+ auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(&R);
+ if (!PhiR || PhiR->getNumIncoming() != 1)
+ continue;
+ PhiR->replaceAllUsesWith(PhiR->getOperand(0));
+ PhiR->eraseFromParent();
+ }
// Disconnect blocks and remove the terminator.
VPBlockUtils::disconnectBlocks(VPBB, RemovedSucc);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 86f709375286e..f8d24f7f08fbc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -76,6 +76,17 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dissolveLoopRegions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::expandBranchOnTwoConds
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToVariableLengthStep
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeBranchOnConst
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeBackedgeTakenCount
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeVectorTripCount
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeFactors
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::expandSCEVsToVPInstructions
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::cse
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeBranchOnConst
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyKnownEVL
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::expandSCEVs
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeDeadRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printFinalVPlan
; Also verify that VPlans are actually printed (we aren't interested in the
More information about the llvm-commits
mailing list