[llvm] [VPlan] Split handleEarlyExits into countable and uncountable passes. NFC (PR #206017)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 01:47:27 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/206017
>From ff58cbe84d6d6d98060fd0d1f41bf91532435b62 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 26 Jun 2026 17:39:53 +0800
Subject: [PATCH 1/3] [VPlan] Split handleEarlyExits into countable and
uncountable passes. NFC
This allows us to remove UncountableExitStyle::NoUncountableExit, and isolate it to just the uncountable exit path. This should make it easier to choose a style from within VPlan alone later on.
This also allows us to plug UncountableExitStyle into a TTI hook or CLI flag eventually, as I don't think we want to expose NoUncoutableExit.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 21 ++++++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 1 -
.../Vectorize/VPlanConstruction.cpp | 35 ++++---------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 14 ++++++--
.../Transforms/Vectorize/VPlanTransforms.h | 21 ++++++++---
.../VPlan/vplan-print-before-after-all.ll | 2 +-
6 files changed, 48 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f1ca4061bfd9e..fa4000494c5d2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6579,15 +6579,18 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// TODO: Remove this once we can properly check the VPlan itself for both
// the presence of an uncountable exit and the presence of stores in
// the loop inside handleEarlyExits itself.
- UncountableExitStyle EEStyle = UncountableExitStyle::NoUncountableExit;
- if (Legal->hasUncountableEarlyExit())
- EEStyle = Legal->hasUncountableExitWithSideEffects()
- ? UncountableExitStyle::MaskedHandleExitInScalarLoop
- : UncountableExitStyle::ReadOnly;
-
- if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
- OrigLoop, PSE, *DT, Legal->getAssumptionCache())) {
- return nullptr;
+ if (Legal->hasUncountableEarlyExit()) {
+ // TODO: Check target preference for style.
+ UncountableExitStyle EEStyle =
+ Legal->hasUncountableExitWithSideEffects()
+ ? UncountableExitStyle::MaskedHandleExitInScalarLoop
+ : UncountableExitStyle::ReadOnly;
+ if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
+ OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
+ EEStyle))
+ return nullptr;
+ } else {
+ RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
}
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eaf9d1433aff7..869cbe1ac49e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -77,7 +77,6 @@ using VPlanPtr = std::unique_ptr<VPlan>;
/// Different methods of handling early exits.
///
enum class UncountableExitStyle {
- NoUncountableExit = 0,
/// No side effects to worry about, so we can process any uncountable exits
/// in the loop and branch either to the middle block if the trip count was
/// reached, or an early exitblock to determine which exit was taken.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 619fea8c10b4d..1cd14df7629aa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1216,12 +1216,11 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
R->eraseFromParent();
}
-/// Check if all loads in the loop are dereferenceable. Iterates over the
-/// loop body blocks reachable from \p HeaderVPBB. Returns false if any
-/// non-dereferenceable load is found.
-static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC) {
ScalarEvolution &SE = *PSE.getSE();
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -1260,29 +1259,8 @@ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
return true;
}
-bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
- Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+void VPlanTransforms::handleCountableEarlyExits(VPlan &Plan) {
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
- auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
-
- // TODO: We would like to detect uncountable exits and stores within loops
- // with such exits from the VPlan alone. Exit detection can be moved
- // here from handleUncountableEarlyExits, but we need to improve
- // detection of recipes which may write to memory.
- if (Style != UncountableExitStyle::NoUncountableExit) {
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
- if (Style == UncountableExitStyle::ReadOnly &&
- !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
- return false;
- // TODO: Check target preference for style.
- return handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB,
- TheLoop, PSE, DT, AC, Style);
- }
-
// Disconnect countable early exits from the loop, leaving it with a single
// exit from the latch. Countable early exits are left for a scalar epilog.
for (VPIRBasicBlock *EB : Plan.getExitBlocks()) {
@@ -1298,7 +1276,6 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
VPBlockUtils::disconnectBlocks(Pred, EB);
}
}
- return true;
}
void VPlanTransforms::addMiddleCheck(VPlan &Plan) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5834f38e96b84..7b413ed84450c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4605,12 +4605,22 @@ static bool handleUncountableExitsWithSideEffects(
}
bool VPlanTransforms::handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
+
+ auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
+ auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
+
+ // Dereferenceability is checked separately for uncountable exit loops with
+ // stores, as only the loads contributing to the exit condition need to
+ // be checked.
+ if (Style == UncountableExitStyle::ReadOnly &&
+ !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
+ return false;
+
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 3260526552281..f873aa1215fab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -357,14 +357,27 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
+ /// Check if all loads in the loop are dereferenceable. Iterates over the
+ /// loop body blocks reachable from \p HeaderVPBB. Returns false if any
+ /// non-dereferenceable load is found.
+ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC);
+
/// Update \p Plan to account for uncountable early exits by introducing
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static bool handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style);
+ static bool handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC,
+ UncountableExitStyle Style);
+
+ /// Disconnect countable early exits from the loop.
+ static void handleCountableEarlyExits(VPlan &Plan);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 2a250d4c896ff..3175f5c89b6d9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -13,7 +13,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::finalizeSCEVPredicates
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMiddleCheck
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleEarlyExits
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleCountableEarlyExits
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createInLoopReductionRecipes
>From e2710052fb354b9cc933a298fa3047607f26d688 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 26 Jun 2026 18:08:42 +0800
Subject: [PATCH 2/3] Fix unit tests
---
.../Transforms/Vectorize/VPlanTestBase.h | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3bcbd3ca6c937..a135fa3a232f2 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -68,10 +68,9 @@ class VPlanTestIRBase : public testing::Test {
}
/// Build the VPlan for the loop starting from \p LoopHeader.
- VPlanPtr buildVPlan(
- BasicBlock *LoopHeader,
- UncountableExitStyle Style = UncountableExitStyle::NoUncountableExit,
- bool CreateLoopRegions = true) {
+ VPlanPtr buildVPlan(BasicBlock *LoopHeader,
+ std::optional<UncountableExitStyle> Style = std::nullopt,
+ bool CreateLoopRegions = true) {
Function &F = *LoopHeader->getParent();
assert(!verifyFunction(F) && "input function must be valid");
doAnalysis(F);
@@ -81,9 +80,9 @@ class VPlanTestIRBase : public testing::Test {
auto Plan =
VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), PSE);
- if (Style != UncountableExitStyle::NoUncountableExit) {
+ if (Style) {
Inductions.clear();
- // handleEarlyExits requires induction phi recipes.
+ // handleUncountableEarlyExits requires induction phi recipes.
for (PHINode &Phi : LoopHeader->phis()) {
InductionDescriptor ID;
if (InductionDescriptor::isInductionPHI(&Phi, L, PSE, ID))
@@ -97,7 +96,11 @@ class VPlanTestIRBase : public testing::Test {
/*AllowReordering=*/false);
}
- VPlanTransforms::handleEarlyExits(*Plan, Style, L, PSE, *DT, AC.get());
+ if (Style)
+ VPlanTransforms::handleUncountableEarlyExits(*Plan, L, PSE, *DT, AC.get(),
+ *Style);
+ else
+ VPlanTransforms::handleCountableEarlyExits(*Plan);
VPlanTransforms::addMiddleCheck(*Plan);
if (CreateLoopRegions)
>From 65d0c160a4e9c359967fe5c8f960b2b3a4fa4fcd Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 19 Aug 2026 16:46:39 +0800
Subject: [PATCH 3/3] Address comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.h | 20 +++++--------------
2 files changed, 6 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7bc3682275a5e..3fc3794aef6c4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6585,7 +6585,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
// the presence of an uncountable exit and the presence of stores in
- // the loop inside handleEarlyExits itself.
+ // the loop inside handleUncountableEarlyExits itself.
if (Legal->hasUncountableEarlyExit()) {
// TODO: Check target preference for style.
UncountableExitStyle EEStyle =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5b7f8e41d458c..c3dd00acfe839 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -188,15 +188,6 @@ struct VPlanTransforms {
/// VPReductionRecipe instances.
static void createInLoopReductionRecipes(VPlan &Plan, ElementCount MinVF);
- /// Update \p Plan to account for all early exits. If \p Style is not
- /// NoUncountableExit, handles uncountable early exits and checks that all
- /// loads are dereferenceable. Returns false if a non-dereferenceable load is
- /// found.
- LLVM_ABI_FOR_TEST static bool
- handleEarlyExits(VPlan &Plan, UncountableExitStyle Style, Loop *TheLoop,
- PredicatedScalarEvolution &PSE, DominatorTree &DT,
- AssumptionCache *AC);
-
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan);
@@ -389,14 +380,13 @@ struct VPlanTransforms {
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static bool handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT,
- AssumptionCache *AC,
- UncountableExitStyle Style);
+ LLVM_ABI_FOR_TEST static bool
+ handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE, DominatorTree &DT,
+ AssumptionCache *AC, UncountableExitStyle Style);
/// Disconnect countable early exits from the loop.
- static void handleCountableEarlyExits(VPlan &Plan);
+ LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
More information about the llvm-commits
mailing list