[llvm] [NFC][LV] Introduce enums for uncountable exit detail and style (PR #184808)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 10 05:02:07 PDT 2026
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/184808
>From 8bad79570167da2f7d8e0a1b0af7ab6dfb303044 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 1 Jul 2025 13:08:48 +0000
Subject: [PATCH 1/3] [NFC][LV] Introduce enums for uncountable exit detail and
style
Recursively splitting out some work from #183318; this covers
the enums for early exit loop type (none, readonly, readwrite)
and the style used (just readonly with multiple exit blocks for
now) and refactoring for basic use of those enums.
---
.../Vectorize/LoopVectorizationLegality.h | 31 ++++++++++++++-----
.../Vectorize/LoopVectorizationLegality.cpp | 11 +++----
.../Transforms/Vectorize/LoopVectorize.cpp | 20 ++++++++++--
llvm/lib/Transforms/Vectorize/VPlan.h | 18 +++++++++++
.../Vectorize/VPlanConstruction.cpp | 10 +++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +++-
.../Transforms/Vectorize/VPlanTransforms.h | 5 +--
.../Transforms/Vectorize/VPlanTestBase.h | 6 ++--
.../Vectorize/VPlanUncountableExitTest.cpp | 4 ++-
9 files changed, 83 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index f82fc588639dd..b5a476b437c40 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -245,6 +245,13 @@ struct HistogramInfo {
: Load(Load), Update(Update), Store(Store) {}
};
+/// Indicates the characteristics of a loop with an uncountable early exit.
+/// * None -- No uncountable exit present.
+/// * ReadOnly -- At least one uncountable exit in a readonly loop.
+/// * ReadWrite -- At least one uncountable exit in a loop with side effects
+/// that may require masking.
+enum class UncountableEarlyExitDetail { None, ReadOnly, ReadWrite };
+
/// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
/// to what vectorization factor.
/// This class does not look at the profitability of vectorization, only the
@@ -407,16 +414,26 @@ class LoopVectorizationLegality {
return LAI->getDepChecker().getMaxSafeVectorWidthInBits();
}
+ /// Returns information about whether this loop contains at least one
+ /// uncountable early exits, and if so, if it also contains instructions
+ /// like stores that cause side-effects.
+ UncountableEarlyExitDetail getUncountableEarlyExitDetail() const {
+ return UncountableExitType;
+ }
+
/// Returns true if the loop has uncountable early exits, i.e. uncountable
/// exits that aren't the latch block.
- bool hasUncountableEarlyExit() const { return HasUncountableEarlyExit; }
+ bool hasUncountableEarlyExit() const {
+ return getUncountableEarlyExitDetail() != UncountableEarlyExitDetail::None;
+ }
/// Returns true if this is an early exit loop with state-changing or
/// potentially-faulting operations and the condition for the uncountable
/// exit must be determined before any of the state changes or potentially
/// faulting operations take place.
bool hasUncountableExitWithSideEffects() const {
- return UncountableExitWithSideEffects;
+ return getUncountableEarlyExitDetail() ==
+ UncountableEarlyExitDetail::ReadWrite;
}
/// Return true if there is store-load forwarding dependencies.
@@ -738,12 +755,10 @@ class LoopVectorizationLegality {
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
- /// True if the loop has uncountable early exits.
- bool HasUncountableEarlyExit = false;
-
- /// If true, the loop has at least one uncountable exit and operations within
- /// the loop may have observable side effects.
- bool UncountableExitWithSideEffects = false;
+ /// Records whether we have an uncountable early exit in a loop that's
+ /// either read-only or read-write.
+ UncountableEarlyExitDetail UncountableExitType =
+ UncountableEarlyExitDetail::None;
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 05f1dc18881df..07a1813d9c3f0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1844,8 +1844,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
LLVM_DEBUG(dbgs() << "LV: Found an early exit loop with symbolic max "
"backedge taken count: "
<< *SymbolicMaxBTC << '\n');
- HasUncountableEarlyExit = true;
- UncountableExitWithSideEffects = HasSideEffects;
+ UncountableExitType = HasSideEffects ? UncountableEarlyExitDetail::ReadWrite
+ : UncountableEarlyExitDetail::ReadOnly;
return true;
}
@@ -2010,8 +2010,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
} else {
if (!isVectorizableEarlyExitLoop()) {
- assert(!hasUncountableEarlyExit() &&
- !hasUncountableExitWithSideEffects() &&
+ assert(UncountableExitType == UncountableEarlyExitDetail::None &&
"Must be false without vectorizable early-exit loop");
if (DoExtraAnalysis)
Result = false;
@@ -2030,8 +2029,8 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- // Bail out for state-changing loops with uncountable exits for now.
- if (UncountableExitWithSideEffects) {
+ // Bail out for ReadWrite loops with uncountable exits for now.
+ if (UncountableExitType == UncountableEarlyExitDetail::ReadWrite) {
reportVectorizationFailure(
"Writes to memory unsupported in early exit loops",
"Cannot vectorize early exit loop with writes to memory",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb4eef5a41c09..9bf82c100c5f3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -785,6 +785,21 @@ createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop,
return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
}
+/// Returns an appropriate style of handling uncountable early exits based on
+/// the exits in the scalar loop. For now it's a 1-1 mapping, but in future
+/// we may choose different styles based on target preference and/or number
+/// of exits in the loop.
+static EarlyExitStyle getEarlyExitStyle(UncountableEarlyExitDetail Detail) {
+ switch (Detail) {
+ case UncountableEarlyExitDetail::None:
+ return EarlyExitStyle::NoUncountableEarlyExit;
+ case UncountableEarlyExitDetail::ReadOnly:
+ return EarlyExitStyle::ReadOnlyUncountableExitsInVectorLoop;
+ case UncountableEarlyExitDetail::ReadWrite:
+ return EarlyExitStyle::MaskedHandleLastIterationInScalarLoop;
+ }
+}
+
namespace llvm {
/// Return a value for Step multiplied by VF.
@@ -8179,7 +8194,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
+ VPlanTransforms::handleEarlyExits(
+ *Plan, getEarlyExitStyle(Legal->getUncountableEarlyExitDetail()));
VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
CM.foldTailByMasking());
@@ -8430,7 +8446,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
/*AllowReordering=*/false);
VPlanTransforms::handleEarlyExits(*Plan,
- /*HasUncountableExit*/ false);
+ EarlyExitStyle::NoUncountableEarlyExit);
VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
/*TailFolded*/ false);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index da2f6f8c7cd03..ae1d6509071b3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -77,6 +77,24 @@ typedef unsigned ID;
using VPlanPtr = std::unique_ptr<VPlan>;
+// Different methods of handling early exits.
+//
+// For ReadOnlyUncountableExitsInVectorLoop, there are no side effects to worry
+// about, so we can instead process any uncountable exit inside the vector
+// body and branch to an appropriate exit block, matching the behaviour of the
+// scalar loop.
+//
+// For MaskedHandleLastIterationInScalarLoop, all memory operations besides the
+// load(s) required to determine whether an uncountable exit occurred will be
+// masked based on that condition. If an uncountable exit is taken, then all
+// lanes before the exiting lane will complete, leaving just the final lane to
+// execute in the scalar tail.
+enum class EarlyExitStyle {
+ NoUncountableEarlyExit = 0,
+ ReadOnlyUncountableExitsInVectorLoop,
+ MaskedHandleLastIterationInScalarLoop
+};
+
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
class LLVM_ABI_FOR_TEST VPBlockBase {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index b9f8cb3da24ff..0fbee693049e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -941,16 +941,14 @@ void VPlanTransforms::createInLoopReductionRecipes(
R->eraseFromParent();
}
-void VPlanTransforms::handleEarlyExits(VPlan &Plan,
- bool HasUncountableEarlyExit) {
+void VPlanTransforms::handleEarlyExits(VPlan &Plan, EarlyExitStyle Style) {
auto *MiddleVPBB = cast<VPBasicBlock>(
Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
- VPBlockBase *HeaderVPB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[1]);
+ VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[1]);
- if (HasUncountableEarlyExit) {
- handleUncountableEarlyExits(Plan, cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
- MiddleVPBB);
+ if (Style != EarlyExitStyle::NoUncountableEarlyExit) {
+ handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB, Style);
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 33e5b25bd9322..be567e804ede7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4097,7 +4097,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB) {
+ VPBasicBlock *MiddleVPBB,
+ EarlyExitStyle Style) {
struct EarlyExitInfo {
VPBasicBlock *EarlyExitingVPBB;
VPIRBasicBlock *EarlyExitVPBB;
@@ -4175,6 +4176,9 @@ void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPValue *IsAnyExitTaken =
Builder.createNaryOp(VPInstruction::AnyOf, {Combined});
+ assert(Style == EarlyExitStyle::ReadOnlyUncountableExitsInVectorLoop &&
+ "Early exit store masking not implemented");
+
// Create the vector.early.exit blocks.
SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 2956659e5df8b..668e35f7a27fd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -151,7 +151,7 @@ struct VPlanTransforms {
/// Update \p Plan to account for all early exits.
LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
- bool HasUncountableExit);
+ EarlyExitStyle Style);
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
@@ -321,7 +321,8 @@ struct VPlanTransforms {
/// that determines which exit to take based on lane-by-lane semantics.
static void handleUncountableEarlyExits(VPlan &Plan, VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB);
+ VPBasicBlock *MiddleVPBB,
+ EarlyExitStyle Style);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3a585e958c3f3..ec7ddbb0a97c5 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -65,7 +65,9 @@ class VPlanTestIRBase : public testing::Test {
}
/// Build the VPlan for the loop starting from \p LoopHeader.
- VPlanPtr buildVPlan(BasicBlock *LoopHeader, bool HasUncountableExit = false) {
+ VPlanPtr
+ buildVPlan(BasicBlock *LoopHeader,
+ EarlyExitStyle Style = EarlyExitStyle::NoUncountableEarlyExit) {
Function &F = *LoopHeader->getParent();
assert(!verifyFunction(F) && "input function must be valid");
doAnalysis(F);
@@ -75,7 +77,7 @@ class VPlanTestIRBase : public testing::Test {
auto Plan = VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64),
{}, PSE);
- VPlanTransforms::handleEarlyExits(*Plan, HasUncountableExit);
+ VPlanTransforms::handleEarlyExits(*Plan, Style);
VPlanTransforms::addMiddleCheck(*Plan, true, false);
VPlanTransforms::createLoopRegions(*Plan);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
index c701382c54e6b..027092b7b61db 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanUncountableExitTest.cpp
@@ -46,7 +46,9 @@ TEST_F(VPUncountableExitTest, FindUncountableExitRecipes) {
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- auto Plan = buildVPlan(LoopHeader, /*HasUncountableExit=*/true);
+ // This is incorrect (wrong style), but will be fixed by a later PR.
+ auto Plan = buildVPlan(LoopHeader,
+ EarlyExitStyle::ReadOnlyUncountableExitsInVectorLoop);
VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI);
VPlanTransforms::optimize(*Plan);
>From c71818db4237611360057b8ad2bc78897324cfaf Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 5 Mar 2026 15:44:49 +0000
Subject: [PATCH 2/3] Improve comment
---
.../llvm/Transforms/Vectorize/LoopVectorizationLegality.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index b5a476b437c40..9f96ac8fe764a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -415,8 +415,8 @@ class LoopVectorizationLegality {
}
/// Returns information about whether this loop contains at least one
- /// uncountable early exits, and if so, if it also contains instructions
- /// like stores that cause side-effects.
+ /// uncountable early exit, and if so, if it also contains instructions (such
+ /// as stores) that cause side-effects.
UncountableEarlyExitDetail getUncountableEarlyExitDetail() const {
return UncountableExitType;
}
>From b63117c7caa144a0ba42d74dcf9387630a002dcd Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 10 Mar 2026 11:50:39 +0000
Subject: [PATCH 3/3] Assert lack of store recipes in a ReadOnly loop
---
.../lib/Transforms/Vectorize/LoopVectorize.cpp | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9bf82c100c5f3..9f3a080cfd3d1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8194,8 +8194,22 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- VPlanTransforms::handleEarlyExits(
- *Plan, getEarlyExitStyle(Legal->getUncountableEarlyExitDetail()));
+
+ UncountableEarlyExitDetail EEDetail = Legal->getUncountableEarlyExitDetail();
+#ifndef NDEBUG
+ // If we're vectorizing a loop with an uncountable exit, make sure that the
+ // recipes are safe to handle.
+ if (EEDetail == UncountableEarlyExitDetail::ReadOnly) {
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Plan->getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT))
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB))
+ if (auto *SDR = dyn_cast<VPSingleDefRecipe>(&R))
+ assert(!match(SDR, m_VPInstruction<Instruction::Store>()) &&
+ "Store recipe found in ReadOnly uncountable exit loop");
+ }
+#endif
+ VPlanTransforms::handleEarlyExits(*Plan, getEarlyExitStyle(EEDetail));
VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
CM.foldTailByMasking());
More information about the llvm-commits
mailing list