[llvm] [VPlan] Compute UncountableExitStyle in VPlan. NFC (PR #206020)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 03:07:02 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Luke Lau (lukel97)
<details>
<summary>Changes</summary>
Stacked on #<!-- -->206017
Currently we choose MaskedHandleExitInScalarLoop basde on a simple Instruction::mayWriteToMemory check in LoopVectorizationLegality. We can just compute this in VPlan from a fairly simple loop over the loop body's recipes. This makes it easier to choose additional uncountable exit styles in future.
---
Full diff: https://github.com/llvm/llvm-project/pull/206020.diff
6 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+9-13)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (+6-29)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+28-3)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+16-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll (+1-1)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 18e6890d13e13..8d127daaf6c09 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6564,24 +6564,20 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// If we're vectorizing a loop with an uncountable exit, make sure that the
// recipes are safe to handle.
- // TODO: Remove this once we can properly check the VPlan itself for both
- // the presence of an uncountable exit and the presence of stores in
- // the loop inside handleEarlyExits itself.
- UncountableExitStyle EEStyle = UncountableExitStyle::NoUncountableExit;
- if (Legal->hasUncountableEarlyExit())
- EEStyle = Legal->hasUncountableExitWithSideEffects()
- ? UncountableExitStyle::MaskedHandleExitInScalarLoop
- : UncountableExitStyle::ReadOnly;
-
- if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
- OrigLoop, PSE, *DT, Legal->getAssumptionCache())) {
- return nullptr;
+ // TODO: Remove this once we can properly check the VPlan itself for the
+ // presence of an uncountable exit.
+ if (Legal->hasUncountableEarlyExit()) {
+ if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
+ OrigLoop, PSE, *DT, Legal->getAssumptionCache()))
+ return nullptr;
+ } else {
+ RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
}
// If we're handling uncountable exits in the scalar tail after a vector
// loop with an in-loop mask, then the middle check has already been
// created to compare against the actual number of lanes executed.
- if (EEStyle != UncountableExitStyle::MaskedHandleExitInScalarLoop)
+ if (!Legal->hasUncountableExitWithSideEffects())
RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bd7f87dbb3a5d..6b2b32c8cc652 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -77,7 +77,6 @@ using VPlanPtr = std::unique_ptr<VPlan>;
/// Different methods of handling early exits.
///
enum class UncountableExitStyle {
- NoUncountableExit = 0,
/// No side effects to worry about, so we can process any uncountable exits
/// in the loop and branch either to the middle block if the trip count was
/// reached, or an early exitblock to determine which exit was taken.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 619fea8c10b4d..1cd14df7629aa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1216,12 +1216,11 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
R->eraseFromParent();
}
-/// Check if all loads in the loop are dereferenceable. Iterates over the
-/// loop body blocks reachable from \p HeaderVPBB. Returns false if any
-/// non-dereferenceable load is found.
-static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC) {
ScalarEvolution &SE = *PSE.getSE();
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -1260,29 +1259,8 @@ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
return true;
}
-bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
- Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC) {
+void VPlanTransforms::handleCountableEarlyExits(VPlan &Plan) {
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
- auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
-
- // TODO: We would like to detect uncountable exits and stores within loops
- // with such exits from the VPlan alone. Exit detection can be moved
- // here from handleUncountableEarlyExits, but we need to improve
- // detection of recipes which may write to memory.
- if (Style != UncountableExitStyle::NoUncountableExit) {
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
- if (Style == UncountableExitStyle::ReadOnly &&
- !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
- return false;
- // TODO: Check target preference for style.
- return handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB,
- TheLoop, PSE, DT, AC, Style);
- }
-
// Disconnect countable early exits from the loop, leaving it with a single
// exit from the latch. Countable early exits are left for a scalar epilog.
for (VPIRBasicBlock *EB : Plan.getExitBlocks()) {
@@ -1298,7 +1276,6 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
VPBlockUtils::disconnectBlocks(Pred, EB);
}
}
- return true;
}
void VPlanTransforms::addMiddleCheck(VPlan &Plan) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cfe21de1c7595..6259952f17469 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4593,13 +4593,38 @@ static bool handleUncountableExitsWithSideEffects(
return true;
}
+/// Returns true if any non-branch recipe in the loop may have side effects.
+static bool loopHasSideEffects(VPBasicBlock *HeaderVPBB) {
+ for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
+ for (VPRecipeBase &R : *VPBB)
+ if (R.mayHaveSideEffects() && &R != VPBB->getTerminator())
+ return true;
+ return false;
+}
+
bool VPlanTransforms::handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
+ VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ DominatorTree &DT, AssumptionCache *AC) {
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
+
+ auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
+ auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
+
+ // TODO: Check target preference for style.
+ UncountableExitStyle Style =
+ loopHasSideEffects(HeaderVPBB)
+ ? UncountableExitStyle::MaskedHandleExitInScalarLoop
+ : UncountableExitStyle::ReadOnly;
+
+ // Dereferenceability is checked separately for uncountable exit loops with
+ // stores, as only the loads contributing to the exit condition need to
+ // be checked.
+ if (Style == UncountableExitStyle::ReadOnly &&
+ !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
+ return false;
+
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 8fbf6793e90c5..59d88f4e470fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -357,14 +357,26 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
+ /// Check if all loads in the loop are dereferenceable. Iterates over the
+ /// loop body blocks reachable from \p HeaderVPBB. Returns false if any
+ /// non-dereferenceable load is found.
+ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
+ Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC);
+
/// Update \p Plan to account for uncountable early exits by introducing
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static bool handleUncountableEarlyExits(
- VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style);
+ static bool handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC);
+
+ /// Disconnect countable early exits from the loop.
+ static void handleCountableEarlyExits(VPlan &Plan);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 86f709375286e..95df7e09d9c8b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -12,7 +12,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::finalizeSCEVPredicates
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleEarlyExits
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::handleCountableEarlyExits
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMiddleCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::introduceMasksAndLinearize
``````````
</details>
https://github.com/llvm/llvm-project/pull/206020
More information about the llvm-commits
mailing list