[llvm] c6a5dd5 - [VPlan] Split handleEarlyExits into countable and uncountable passes. NFC (#206017)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 06:16:29 PDT 2026
Author: Luke Lau
Date: 2026-08-19T13:16:24Z
New Revision: c6a5dd5bfe32bde0d13426981592ed68694360c4
URL: https://github.com/llvm/llvm-project/commit/c6a5dd5bfe32bde0d13426981592ed68694360c4
DIFF: https://github.com/llvm/llvm-project/commit/c6a5dd5bfe32bde0d13426981592ed68694360c4.diff
LOG: [VPlan] Split handleEarlyExits into countable and uncountable passes. NFC (#206017)
This allows us to remove UncountableExitStyle::NoUncountableExit, and
isolate it to just the uncountable exit path. This should make it easier
to choose a style from within VPlan alone later on.
This also allows us to plug UncountableExitStyle into a TTI hook or CLI
flag eventually, as I don't think we want to expose NoUncountableExit.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb8e560b4ef20..d6562cd5de3b0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6583,16 +6583,19 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
// the presence of an uncountable exit and the presence of stores in
- // the loop inside handleEarlyExits itself.
- UncountableExitStyle EEStyle = UncountableExitStyle::NoUncountableExit;
- if (Legal->hasUncountableEarlyExit())
- EEStyle = Legal->hasUncountableExitWithSideEffects()
- ? UncountableExitStyle::MaskedHandleExitInScalarLoop
- : UncountableExitStyle::ReadOnly;
-
- if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
- OrigLoop, PSE, *DT, Legal->getAssumptionCache())) {
- return nullptr;
+ // the loop inside handleUncountableEarlyExits itself.
+ if (Legal->hasUncountableEarlyExit()) {
+ // TODO: Check target preference for style.
+ UncountableExitStyle EEStyle =
+ Legal->hasUncountableExitWithSideEffects()
+ ? UncountableExitStyle::MaskedHandleExitInScalarLoop
+ : UncountableExitStyle::ReadOnly;
+ if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
+ OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
+ EEStyle))
+ return nullptr;
+ } else {
+ RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
}
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 40c33de7504e7..f91c66af9b5c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -77,7 +77,6 @@ using VPlanPtr = std::unique_ptr<VPlan>;
/// Different methods of handling early exits.
///
enum class UncountableExitStyle {
- NoUncountableExit = 0,
/// No side effects to worry about, so we can process any uncountable exits
/// in the loop and branch either to the middle block if the trip count was
/// reached, or an early exitblock to determine which exit was taken.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index aa3584f755617..a81feb7077651 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1199,12 +1199,11 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
R->eraseFromParent();
}
-/// Check if all loads in the loop are dereferenceable. Iterates over the
-/// loop body blocks reachable from \p HeaderVPBB. Returns false if any
-/// non-dereferenceable load is found.
-static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC) {
ScalarEvolution &SE = *PSE.getSE();
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -1243,29 +1242,8 @@ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
return true;
}
-bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
- Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+void VPlanTransforms::handleCountableEarlyExits(VPlan &Plan) {
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
- auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
-
- // TODO: We would like to detect uncountable exits and stores within loops
- // with such exits from the VPlan alone. Exit detection can be moved
- // here from handleUncountableEarlyExits, but we need to improve
- // detection of recipes which may write to memory.
- if (Style != UncountableExitStyle::NoUncountableExit) {
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
- if (Style == UncountableExitStyle::ReadOnly &&
- !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
- return false;
- // TODO: Check target preference for style.
- return handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB,
- TheLoop, PSE, DT, AC, Style);
- }
-
// Disconnect countable early exits from the loop, leaving it with a single
// exit from the latch. Countable early exits are left for a scalar epilog.
for (auto [EarlyExitingVPBB, EB] : vputils::getEarlyExits(Plan, MiddleVPBB)) {
@@ -1275,7 +1253,6 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
EarlyExitingVPBB->getTerminator()->eraseFromParent();
VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EB);
}
- return true;
}
void VPlanTransforms::addMiddleCheck(VPlan &Plan) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3abfbf5cfb444..0f76041a9cd7c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3209,12 +3209,22 @@ static bool handleUncountableExitsWithSideEffects(
}
bool VPlanTransforms::handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
+
+ auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
+ auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
+
+ // Dereferenceability is checked separately for uncountable exit loops with
+ // stores, as only the loads contributing to the exit condition need to
+ // be checked.
+ if (Style == UncountableExitStyle::ReadOnly &&
+ !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
+ return false;
+
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
for (auto [EarlyExitingVPBB, ExitBlock] :
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 184718186b190..c3dd00acfe839 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -188,15 +188,6 @@ struct VPlanTransforms {
/// VPReductionRecipe instances.
static void createInLoopReductionRecipes(VPlan &Plan, ElementCount MinVF);
- /// Update \p Plan to account for all early exits. If \p Style is not
- /// NoUncountableExit, handles uncountable early exits and checks that all
- /// loads are dereferenceable. Returns false if a non-dereferenceable load is
- /// found.
- LLVM_ABI_FOR_TEST static bool
- handleEarlyExits(VPlan &Plan, UncountableExitStyle Style, Loop *TheLoop,
- PredicatedScalarEvolution &PSE, DominatorTree &DT,
- AssumptionCache *AC);
-
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan);
@@ -376,14 +367,26 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
+ /// Check if all loads in the loop are dereferenceable. Iterates over the
+ /// loop body blocks reachable from \p HeaderVPBB. Returns false if any
+ /// non-dereferenceable load is found.
+ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC);
+
/// Update \p Plan to account for uncountable early exits by introducing
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static bool handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style);
+ LLVM_ABI_FOR_TEST static bool
+ handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE, DominatorTree &DT,
+ AssumptionCache *AC, UncountableExitStyle Style);
+
+ /// Disconnect countable early exits from the loop.
+ LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 6b2687de4d53f..1afd9a1955028 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -18,7 +18,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::finalizeSCEVPredicates
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMiddleCheck
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleEarlyExits
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleCountableEarlyExits
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createInLoopReductionRecipes
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3bcbd3ca6c937..a135fa3a232f2 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -68,10 +68,9 @@ class VPlanTestIRBase : public testing::Test {
}
/// Build the VPlan for the loop starting from \p LoopHeader.
- VPlanPtr buildVPlan(
- BasicBlock *LoopHeader,
- UncountableExitStyle Style = UncountableExitStyle::NoUncountableExit,
- bool CreateLoopRegions = true) {
+ VPlanPtr buildVPlan(BasicBlock *LoopHeader,
+ std::optional<UncountableExitStyle> Style = std::nullopt,
+ bool CreateLoopRegions = true) {
Function &F = *LoopHeader->getParent();
assert(!verifyFunction(F) && "input function must be valid");
doAnalysis(F);
@@ -81,9 +80,9 @@ class VPlanTestIRBase : public testing::Test {
auto Plan =
VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), PSE);
- if (Style != UncountableExitStyle::NoUncountableExit) {
+ if (Style) {
Inductions.clear();
- // handleEarlyExits requires induction phi recipes.
+ // handleUncountableEarlyExits requires induction phi recipes.
for (PHINode &Phi : LoopHeader->phis()) {
InductionDescriptor ID;
if (InductionDescriptor::isInductionPHI(&Phi, L, PSE, ID))
@@ -97,7 +96,11 @@ class VPlanTestIRBase : public testing::Test {
/*AllowReordering=*/false);
}
- VPlanTransforms::handleEarlyExits(*Plan, Style, L, PSE, *DT, AC.get());
+ if (Style)
+ VPlanTransforms::handleUncountableEarlyExits(*Plan, L, PSE, *DT, AC.get(),
+ *Style);
+ else
+ VPlanTransforms::handleCountableEarlyExits(*Plan);
VPlanTransforms::addMiddleCheck(*Plan);
if (CreateLoopRegions)
More information about the llvm-commits
mailing list