[llvm] [LV] Move predication, early exit & region handling to VPlan0 (NFCI) (PR #185305)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 11 03:28:13 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/185305
>From cdfaf8af03c83f185a4f106a99256138594c007a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 8 Mar 2026 15:49:10 +0000
Subject: [PATCH 1/2] [LV] Move predication, early exit & region handling to
VPlan0 (NFCI)
Move handleEarlyExits, predication and region creation to operate
directly on VPlan0. This means they only have to run once, reducing
compile time a bit; the relative order remains unchanged.
Introducing the regions at this point in particular unlocks performing
more transforms once, on the initial VPlan, instead of running them for
each VF.
Whether a scalar epilogue is required is still determined by legacy cost
model, so we need to still account for that in the VF specific VPlan
logic.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 31 +++++++++++--------
1 file changed, 18 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 420b111bafc72..58abd3e42c52c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8160,6 +8160,15 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
CM.getInLoopReductions(), Hints.allowReordering());
VPlanTransforms::simplifyRecipes(*VPlan0);
+ VPlanTransforms::handleEarlyExits(*VPlan0, Legal->hasUncountableEarlyExit());
+ VPlanTransforms::addMiddleCheck(*VPlan0,
+ /*RequiresScalarEpilogueCheck=*/true,
+ CM.foldTailByMasking());
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *VPlan0);
+ if (CM.foldTailByMasking())
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *VPlan0);
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize,
+ *VPlan0);
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
@@ -8206,13 +8215,15 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
- VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
- CM.foldTailByMasking());
-
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *Plan);
- if (CM.foldTailByMasking())
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *Plan);
+ // Update the branch in the middle block if a scalar epilogue is required.
+ auto *MiddleVPBB = Plan->getMiddleBlock();
+ if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
+ auto *MiddleVPBB = Plan->getMiddleBlock();
+ auto *BranchOnCond = cast<VPInstruction>(MiddleVPBB->getTerminator());
+ assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
+ "second successor must be scalar preheader");
+ BranchOnCond->setOperand(0, Plan->getFalse());
+ }
// Don't use getDecisionAndClampRange here, because we don't know the UF
// so this function is better to be conservative, rather than to split
@@ -8264,11 +8275,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
- // ---------------------------------------------------------------------------
- // Predicate and linearize the top-level loop region.
- // ---------------------------------------------------------------------------
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize, *Plan);
-
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
@@ -8281,7 +8287,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
HeaderVPBB);
- auto *MiddleVPBB = Plan->getMiddleBlock();
VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
// Collect blocks that need predication for in-loop reduction recipes.
>From 351825ef7e8ce49e9b65725d087e786eb9db2715 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 11 Mar 2026 10:15:22 +0000
Subject: [PATCH 2/2] !fiuxp remove requiresscalarepilouge arg
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++-----
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 8 ++------
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 4 +---
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h | 2 +-
4 files changed, 6 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7aee09d47d131..1ac6794203025 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8134,9 +8134,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
VPlanTransforms::simplifyRecipes(*VPlan0);
VPlanTransforms::handleEarlyExits(*VPlan0, Legal->hasUncountableEarlyExit());
- VPlanTransforms::addMiddleCheck(*VPlan0,
- /*RequiresScalarEpilogueCheck=*/true,
- CM.foldTailByMasking());
+ VPlanTransforms::addMiddleCheck(*VPlan0, CM.foldTailByMasking());
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *VPlan0);
if (CM.foldTailByMasking())
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *VPlan0);
@@ -8436,8 +8434,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
/*AllowReordering=*/false);
VPlanTransforms::handleEarlyExits(*Plan,
/*HasUncountableExit*/ false);
- VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
- /*TailFolded*/ false);
+ VPlanTransforms::addMiddleCheck(*Plan, /*TailFolded*/ false);
VPlanTransforms::createLoopRegions(*Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index b9f8cb3da24ff..56290fc15dc57 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -971,9 +971,7 @@ void VPlanTransforms::handleEarlyExits(VPlan &Plan,
}
}
-void VPlanTransforms::addMiddleCheck(VPlan &Plan,
- bool RequiresScalarEpilogueCheck,
- bool TailFolded) {
+void VPlanTransforms::addMiddleCheck(VPlan &Plan, bool TailFolded) {
auto *MiddleVPBB = cast<VPBasicBlock>(
Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
// If MiddleVPBB has a single successor then the original loop does not exit
@@ -1006,9 +1004,7 @@ void VPlanTransforms::addMiddleCheck(VPlan &Plan,
DebugLoc LatchDL = LatchVPBB->getTerminator()->getDebugLoc();
VPBuilder Builder(MiddleVPBB);
VPValue *Cmp;
- if (!RequiresScalarEpilogueCheck)
- Cmp = Plan.getFalse();
- else if (TailFolded)
+ if (TailFolded)
Cmp = Plan.getTrue();
else
Cmp = Builder.createICmp(CmpInst::ICMP_EQ, Plan.getTripCount(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5f060b32da847..d10ef23dd05b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -155,9 +155,7 @@ struct VPlanTransforms {
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
- LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan,
- bool RequiresScalarEpilogueCheck,
- bool TailFolded);
+ LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan, bool TailFolded);
// Create a check to \p Plan to see if the vector loop should be executed.
static void addMinimumIterationCheck(
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3a585e958c3f3..7ab934feb9758 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -76,7 +76,7 @@ class VPlanTestIRBase : public testing::Test {
{}, PSE);
VPlanTransforms::handleEarlyExits(*Plan, HasUncountableExit);
- VPlanTransforms::addMiddleCheck(*Plan, true, false);
+ VPlanTransforms::addMiddleCheck(*Plan, false);
VPlanTransforms::createLoopRegions(*Plan);
return Plan;
More information about the llvm-commits
mailing list