[llvm] [LV] Allow loops with multiple early exits in legality checks. (PR #176403)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 19 00:36:15 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/176403
>From efe6641a4817df945a7d491476a7c9ce9e537fd0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 16 Jan 2026 13:16:54 +0000
Subject: [PATCH 1/2] [LV] Allow loops with multiple early exits in legality
checks.
This patch removes the single uncountable exit constraint, allowing
loops with multiple early exits, if the exits form a dominance chain and
all other constraints hold for all uncountable early exits.
While legality now accepts such loops, vectorization is not yet
supported. VPlan support will be added in a follow up:
https://github.com/llvm/llvm-project/pull/174864
---
.../Vectorize/LoopVectorizationLegality.h | 16 ++---
.../Vectorize/LoopVectorizationLegality.cpp | 63 ++++++++++---------
.../Transforms/Vectorize/LoopVectorize.cpp | 21 +++++--
.../LoopVectorize/early_exit_legality.ll | 8 ++-
.../early_exit_store_legality.ll | 2 +-
5 files changed, 62 insertions(+), 48 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 7850d9c70252a..f82fc588639dd 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -407,14 +407,9 @@ class LoopVectorizationLegality {
return LAI->getDepChecker().getMaxSafeVectorWidthInBits();
}
- /// Returns true if the loop has exactly one uncountable early exit, i.e. an
- /// uncountable exit that isn't the latch block.
- bool hasUncountableEarlyExit() const { return UncountableExitingBB; }
-
- /// Returns the uncountable early exiting block, if there is exactly one.
- BasicBlock *getUncountableEarlyExitingBlock() const {
- return UncountableExitingBB;
- }
+ /// Returns true if the loop has uncountable early exits, i.e. uncountable
+ /// exits that aren't the latch block.
+ bool hasUncountableEarlyExit() const { return HasUncountableEarlyExit; }
/// Returns true if this is an early exit loop with state-changing or
/// potentially-faulting operations and the condition for the uncountable
@@ -743,9 +738,8 @@ class LoopVectorizationLegality {
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
- /// Keep track of an uncountable exiting block, if there is exactly one early
- /// exit.
- BasicBlock *UncountableExitingBB = nullptr;
+ /// True if the loop has uncountable early exits.
+ bool HasUncountableEarlyExit = false;
/// If true, the loop has at least one uncountable exit and operations within
/// the loop may have observable side effects.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 93229ea625a5d..f179ae6f63bdc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -25,6 +25,7 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
@@ -1434,13 +1435,10 @@ bool LoopVectorizationLegality::isFixedOrderRecurrence(
bool LoopVectorizationLegality::blockNeedsPredication(
const BasicBlock *BB) const {
// When vectorizing early exits, create predicates for the latch block only.
- // The early exiting block must be a direct predecessor of the latch at the
- // moment.
+ // For a single early exit, it must be a direct predecessor of the latch.
+ // For multiple early exits, they form a chain leading to the latch.
BasicBlock *Latch = TheLoop->getLoopLatch();
if (hasUncountableEarlyExit()) {
- assert(
- is_contained(predecessors(Latch), getUncountableEarlyExitingBlock()) &&
- "Uncountable exiting block must be a direct predecessor of latch");
return BB == Latch;
}
return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
@@ -1719,7 +1717,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Keep a record of all the exiting blocks.
SmallVector<const SCEVPredicate *, 4> Predicates;
- BasicBlock *SingleUncountableExitingBlock = nullptr;
+ SmallVector<BasicBlock *> UncountableExitingBlocks;
for (BasicBlock *BB : ExitingBlocks) {
const SCEV *EC =
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
@@ -1732,15 +1730,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
return false;
}
- if (SingleUncountableExitingBlock) {
- reportVectorizationFailure(
- "Loop has too many uncountable exits",
- "Cannot vectorize early exit loop with more than one early exit",
- "TooManyUncountableEarlyExits", ORE, TheLoop);
- return false;
- }
-
- SingleUncountableExitingBlock = BB;
+ UncountableExitingBlocks.push_back(BB);
} else
CountableExitingBlocks.push_back(BB);
}
@@ -1750,15 +1740,32 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// PSE.getSymbolicMaxBackedgeTakenCount() below.
Predicates.clear();
- if (!SingleUncountableExitingBlock) {
- LLVM_DEBUG(dbgs() << "LV: Cound not find any uncountable exits");
+ if (UncountableExitingBlocks.empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Could not find any uncountable exits");
return false;
}
- // The only supported early exit loops so far are ones where the early
- // exiting block is a unique predecessor of the latch block.
+ // Sort exiting blocks by dominance order to establish a clear chain.
+ llvm::sort(UncountableExitingBlocks, [this](BasicBlock *A, BasicBlock *B) {
+ return DT->properlyDominates(A, B);
+ });
+
+ // Verify that exits form a strict dominance chain: each block must
+ // dominate the next. This ensures each exit is only dominated by its
+ // predecessors in the chain.
+ for (unsigned I = 0; I + 1 < UncountableExitingBlocks.size(); ++I) {
+ if (!DT->properlyDominates(UncountableExitingBlocks[I],
+ UncountableExitingBlocks[I + 1])) {
+ reportVectorizationFailure(
+ "Uncountable early exits do not form a dominance chain",
+ "Cannot vectorize early exit loop with non-dominating exits",
+ "NonDominatingEarlyExits", ORE, TheLoop);
+ return false;
+ }
+ }
+
BasicBlock *LatchPredBB = LatchBB->getUniquePredecessor();
- if (LatchPredBB != SingleUncountableExitingBlock) {
+ if (LatchPredBB != UncountableExitingBlocks.back()) {
reportVectorizationFailure("Early exit is not the latch predecessor",
"Cannot vectorize early exit loop",
"EarlyExitNotLatchPredecessor", ORE, TheLoop);
@@ -1818,10 +1825,6 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
}
- // The vectoriser cannot handle loads that occur after the early exit block.
- assert(LatchBB->getUniquePredecessor() == SingleUncountableExitingBlock &&
- "Expected latch predecessor to be the early exiting block");
-
SmallVector<LoadInst *, 4> NonDerefLoads;
// TODO: Handle loops that may fault.
if (!HasSideEffects) {
@@ -1834,9 +1837,13 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
"NonReadOnlyEarlyExitLoop", ORE, TheLoop);
return false;
}
- } else if (!canUncountableExitConditionLoadBeMoved(
- SingleUncountableExitingBlock))
- return false;
+ } else {
+ // Check all uncountable exiting blocks for movable loads.
+ for (BasicBlock *ExitingBB : UncountableExitingBlocks) {
+ if (!canUncountableExitConditionLoadBeMoved(ExitingBB))
+ return false;
+ }
+ }
// Check non-dereferenceable loads if any.
for (LoadInst *LI : NonDerefLoads) {
@@ -1864,7 +1871,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
LLVM_DEBUG(dbgs() << "LV: Found an early exit loop with symbolic max "
"backedge taken count: "
<< *SymbolicMaxBTC << '\n');
- UncountableExitingBB = SingleUncountableExitingBlock;
+ HasUncountableEarlyExit = true;
UncountableExitWithSideEffects = HasSideEffects;
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ce0aab1e2bde8..f45b146c1294a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9761,11 +9761,22 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}
- if (LVL.hasUncountableEarlyExit() && !EnableEarlyExitVectorization) {
- reportVectorizationFailure("Auto-vectorization of loops with uncountable "
- "early exit is not enabled",
- "UncountableEarlyExitLoopsDisabled", ORE, L);
- return false;
+ if (LVL.hasUncountableEarlyExit()) {
+ if (!EnableEarlyExitVectorization) {
+ reportVectorizationFailure("Auto-vectorization of loops with uncountable "
+ "early exit is not enabled",
+ "UncountableEarlyExitLoopsDisabled", ORE, L);
+ return false;
+ }
+ SmallVector<BasicBlock *, 8> ExitingBlocks;
+ L->getExitingBlocks(ExitingBlocks);
+ // TODO: Support multiple uncountable early exits.
+ if (ExitingBlocks.size() - LVL.getCountableExitingBlocks().size() > 1) {
+ reportVectorizationFailure("Auto-vectorization of loops with multiple "
+ "uncountable early exits is not yet supported",
+ "MultipleUncountableEarlyExits", ORE, L);
+ return false;
+ }
}
if (!LVL.getPotentiallyFaultingLoads().empty()) {
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index 81b10e02107ff..32c4f128423b8 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -346,10 +346,12 @@ loop.end:
}
-; We don't currently support multiple uncountable early exits.
+; Multiple uncountable early exits pass legality but are not yet supported
+; in VPlan transformations.
define i64 @multiple_uncountable_exits() {
; CHECK-LABEL: LV: Checking a loop in 'multiple_uncountable_exits'
-; CHECK: LV: Not vectorizing: Loop has too many uncountable exits.
+; CHECK: LV: We can vectorize this loop!
+; CHECK: LV: Not vectorizing: Auto-vectorization of loops with multiple uncountable early exits is not yet supported.
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -596,7 +598,7 @@ loop.end:
; Two early exits on parallel branches (neither dominates the other).
define i64 @uncountable_exits_on_parallel_branches() {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exits_on_parallel_branches'
-; CHECK: LV: Not vectorizing: Loop has too many uncountable exits.
+; CHECK: LV: Not vectorizing: Uncountable early exits do not form a dominance chain.
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 4e8b8e51df6c2..d16a1435dafbf 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -633,7 +633,7 @@ invalid.block:
define void @crash_conditional_load_for_uncountable_exit_argptr(ptr dereferenceable(40) noalias %store.area, ptr dereferenceable(4) %load.area, i1 %skip.cond) {
; CHECK-LABEL: LV: Checking a loop in 'crash_conditional_load_for_uncountable_exit_argptr'
-; CHECK: LV: Not vectorizing: Loop has too many uncountable exits.
+; CHECK: LV: Not vectorizing: Early exit loop with store but no supported condition load.
entry:
br label %for.body
>From c1bad3a9a96c554e153589145f40779335c98abb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 19 Jan 2026 08:35:44 +0000
Subject: [PATCH 2/2] !fixup addres scomments, update messages, thanks
---
.../Vectorize/LoopVectorizationLegality.cpp | 13 +++++++------
.../Transforms/LoopVectorize/early_exit_legality.ll | 4 ++--
.../LoopVectorize/early_exit_store_legality.ll | 2 +-
.../LoopVectorize/uncountable-early-exit-vplan.ll | 2 +-
4 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index f179ae6f63bdc..ea431b4fa5df8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1436,11 +1436,11 @@ bool LoopVectorizationLegality::blockNeedsPredication(
const BasicBlock *BB) const {
// When vectorizing early exits, create predicates for the latch block only.
// For a single early exit, it must be a direct predecessor of the latch.
- // For multiple early exits, they form a chain leading to the latch.
+ // For multiple early exits, they form a chain where each exiting block
+ // dominates all subsequent blocks up to the latch.
BasicBlock *Latch = TheLoop->getLoopLatch();
- if (hasUncountableEarlyExit()) {
+ if (hasUncountableEarlyExit())
return BB == Latch;
- }
return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
}
@@ -1766,9 +1766,10 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
BasicBlock *LatchPredBB = LatchBB->getUniquePredecessor();
if (LatchPredBB != UncountableExitingBlocks.back()) {
- reportVectorizationFailure("Early exit is not the latch predecessor",
- "Cannot vectorize early exit loop",
- "EarlyExitNotLatchPredecessor", ORE, TheLoop);
+ reportVectorizationFailure(
+ "Last early exiting block in the chain is not the latch predecessor",
+ "Cannot vectorize early exit loop", "EarlyExitNotLatchPredecessor", ORE,
+ TheLoop);
return false;
}
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index 32c4f128423b8..6c35417bd4492 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -318,7 +318,7 @@ return:
; support this yet.
define i64 @uncountable_exit_on_last_block() {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_on_last_block'
-; CHECK: LV: Not vectorizing: Early exit is not the latch predecessor.
+; CHECK: LV: Not vectorizing: Last early exiting block in the chain is not the latch predecessor.
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -496,7 +496,7 @@ exit: ; preds = %for.body
define i64 @uncountable_exit_in_conditional_block(ptr %mask) {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_in_conditional_block'
-; CHECK: LV: Not vectorizing: Early exit is not the latch predecessor.
+; CHECK: LV: Not vectorizing: Last early exiting block in the chain is not the latch predecessor.
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index d16a1435dafbf..55b52299d4331 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -461,7 +461,7 @@ exit:
define void @loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute'
-; CHECK: LV: Not vectorizing: Early exit is not the latch predecessor.
+; CHECK: LV: Not vectorizing: Last early exiting block in the chain is not the latch predecessor.
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
index f279a97a9a067..be23acd443229 100644
--- a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
@@ -241,7 +241,7 @@ exit:
}
define i64 @two_early_exits_same_exit_with_constant_live_outs() {
-; CHECK: LV: Not vectorizing: Loop has too many uncountable exits.
+; CHECK: LV: Not vectorizing: Auto-vectorization of loops with multiple uncountable early exits is not yet supported.
;
entry:
%A = alloca [1024 x i8]
More information about the llvm-commits
mailing list