[llvm] [VPlan] Preserve SSA in early exits (PR #201784)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 05:48:38 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/201784
>From 755f5e63b2b58a4bb4c5d7c88051e957b585157a Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 16 Sep 2026 21:34:12 +0800
Subject: [PATCH 1/4] Precommit test
---
.../LoopVectorize/VPlan/early-exit.ll | 91 +++++++++++++++++++
1 file changed, 91 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
new file mode 100644
index 0000000000000..27bbbeb16fa61
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -disable-output -vplan-print-after=handleUncountableEarlyExits -vplan-verify-each -disable-output %s 2>&1 | FileCheck %s
+
+define i64 @single_exit_in_conditional_block(ptr dereferenceable(64) %a, ptr dereferenceable(64) %b) {
+; CHECK-LABEL: VPlan for loop in 'single_exit_in_conditional_block'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<64> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.A> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%l.A> = load ir<%gep.A>
+; CHECK-NEXT: EMIT ir<%branch.cond> = icmp slt ir<%l.A>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond ir<%branch.cond> (!vplan.prof.estimated estimated {805306368, 1342177280})
+; CHECK-NEXT: Successor(s): block.a, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: block.a:
+; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
+; CHECK-NEXT: EMIT-SCALAR ir<%l.B> = load ir<%gep.B> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
+; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%l.A>, ir<%l.B> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = masked-cond ir<%cmp>
+; CHECK-NEXT: Successor(s): loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> (!vplan.execution.frequency 9115285645797883904 (98.83%, estimated))
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<64> (!vplan.execution.frequency 9115285645797883904 (98.83%, estimated))
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = freeze vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = any-of vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP4]]>, ir<%ec>
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<64>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<loop.end>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<%first.active.lane> = first-active-lane vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<%early.exit.value> = extract-lane vp<%first.active.lane>, ir<%iv>
+; CHECK-NEXT: Successor(s): ir-bb<loop.end>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.end>:
+; CHECK-NEXT: IR %retval = phi i64 [ %iv, %block.a ], [ -1, %loop.latch ] (extra operands: ir<-1> from middle.block, vp<%early.exit.value> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %a, i64 %iv
+; CHECK-NEXT: IR %l.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %branch.cond = icmp slt i8 %l.A, 0
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %a, i64 %iv
+ %l.A = load i8, ptr %gep.A, align 1
+ %branch.cond = icmp slt i8 %l.A, 0
+ br i1 %branch.cond, label %block.a, label %loop.latch
+
+block.a:
+ %gep.B = getelementptr inbounds i8, ptr %b, i64 %iv
+ %l.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp eq i8 %l.A, %l.B
+ br i1 %cmp, label %loop.end, label %loop.latch
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, 64
+ br i1 %ec, label %loop.header, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %iv, %block.a ], [ -1, %loop.latch ]
+ ret i64 %retval
+}
>From be73842ce24d93b9e0cef05e879b0067b5dd2f6e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 11 Sep 2026 17:42:36 +0800
Subject: [PATCH 2/4] [VPlan] Maintain SSA in early exits
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 ++++++++++---------
.../Transforms/Vectorize/VPlanVerifier.cpp | 18 --------
.../LoopVectorize/VPlan/early-exit.ll | 2 +-
.../predicated-early-exits-interleave.ll | 18 ++++----
.../predicated-multiple-exits.ll | 33 ++++++---------
5 files changed, 45 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a2f2a2997086e..a1b4aebf9fd7e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3040,14 +3040,13 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
// EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
// EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
- // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
// Successor(s): for.inc
//
// for.inc:
// EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
// EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
// EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
- // EMIT vp<%freeze> = freeze ir<%3>
+ // EMIT vp<%freeze> = freeze ir<%uncountable.cond>
// EMIT vp<%4> = any-of ir<%freeze>
// EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
// EMIT branch-on-two-conds vp<%4>, vp<%5>
@@ -3101,12 +3100,9 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
return nullptr;
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
Recipes.push_back(cast<VPInstruction>(GepR));
- } else if (match(V, m_Freeze(m_VPValue(
- Op1, m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op2)))))) {
- Worklist.push_back(Op2);
+ } else if (match(V, m_Freeze(m_VPValue(Op1)))) {
+ Worklist.push_back(Op1);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
- Recipes.push_back(cast<VPInstruction>(Op1->getDefiningRecipe()));
} else
return nullptr;
}
@@ -3137,7 +3133,6 @@ struct EarlyExitInfo {
/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
-/// EMIT vp<%1> = masked-cond ir<%cmp1>
/// Successor(s): if.end
///
/// if.end:
@@ -3337,20 +3332,20 @@ bool VPlanTransforms::handleUncountableEarlyExits(
m_BranchOnCond(m_VPValue(CondOfEarlyExitingVPBB)));
assert(Matched && "Terminator must be BranchOnCond");
- // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
- // the correct block mask.
VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
- auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
- VPInstruction::MaskedCond,
+ auto *CondToEarlyExit =
TrueSucc == ExitBlock
? CondOfEarlyExitingVPBB
- : EarlyExitingBuilder.createNot(CondOfEarlyExitingVPBB));
- assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
- !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
- VPDT.properlyDominates(
- CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
- LatchVPBB)) &&
- "exit condition must dominate the latch");
+ : EarlyExitingBuilder.createNot(CondOfEarlyExitingVPBB);
+
+ // Add phis so there's a def of CondToEarlyExit on every path leading to the
+ // latch. The condition is false on paths that didn't go through
+ // EarlyExitingVPBB. EarlyExitingVPBB may be the same as HeaderVPBB, so
+ // assign in order.
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{HeaderVPBB, Plan.getFalse()}};
+ Defs[EarlyExitingVPBB] = CondToEarlyExit;
+ CondToEarlyExit = vputils::reconstructSSA(LatchVPBB, Defs);
+
Exits.push_back({
EarlyExitingVPBB,
ExitBlock,
@@ -3489,6 +3484,15 @@ bool VPlanTransforms::handleUncountableEarlyExits(
ExitIRI->getIncomingValueForBlock(EarlyExitingVPBB);
VPValue *NewIncoming = IncomingVal;
if (!isa<VPIRValue>(IncomingVal)) {
+ // Add phis so IncomingVal is defined on all paths to the latch.
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {HeaderVPBB, Plan.getPoison(IncomingVal->getScalarType())}};
+ VPBasicBlock *DefVPBB = IncomingVal->getDefiningRecipe()->getParent();
+ assert(VPDT.dominates(HeaderVPBB, DefVPBB) &&
+ "IncomingVal defined outside of vector body?");
+ Defs[DefVPBB] = IncomingVal;
+ IncomingVal = vputils::reconstructSSA(LatchVPBB, Defs);
+
VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
NewIncoming = EarlyExitBuilder.createNaryOp(
VPInstruction::ExtractLane, {FirstActiveLane, IncomingVal},
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index e36ad81cfae2a..66509831d08e6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -248,11 +248,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
return false;
}
- // MaskedCond may be used from blocks it don't dominate; the block will be
- // linearized and it will dominate its users after linearization.
- if (match(&R, m_VPInstruction<VPInstruction::MaskedCond>()))
- continue;
-
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
if (isa<VPIRPhi>(UI) &&
@@ -297,19 +292,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
continue;
}
- // Recipes in blocks with a MaskedCond may be used in exit blocks; the
- // block will be linearized and its recipes will dominate their users
- // after linearization.
- bool BlockHasMaskedCond = any_of(*VPBB, [](const VPRecipeBase &R) {
- return match(&R, m_VPInstruction<VPInstruction::MaskedCond>());
- });
- if (BlockHasMaskedCond &&
- any_of(VPBB->getPlan()->getExitBlocks(), [UI](VPIRBasicBlock *EB) {
- return is_contained(EB->getPredecessors(), UI->getParent());
- })) {
- continue;
- }
-
errs() << "Use before def!\n";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
VPSlotTracker Tracker(VPBB->getPlan());
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
index 27bbbeb16fa61..3467f434b2fcb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early-exit.ll
@@ -26,10 +26,10 @@ define i64 @single_exit_in_conditional_block(ptr dereferenceable(64) %a, ptr der
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
; CHECK-NEXT: EMIT-SCALAR ir<%l.B> = load ir<%gep.B> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%l.A>, ir<%l.B> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
-; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = masked-cond ir<%cmp>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = phi [ ir<%cmp>, block.a ], [ ir<false>, loop.header ]
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> (!vplan.execution.frequency 9115285645797883904 (98.83%, estimated))
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<64> (!vplan.execution.frequency 9115285645797883904 (98.83%, estimated))
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = freeze vp<[[VP2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll b/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
index df24b2838f6c4..d07ea380f55b8 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
@@ -185,10 +185,8 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[GEP_A]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
-; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD1]], splat (i8 -42)
-; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
-; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sge <4 x i8> [[WIDE_LOAD1]], splat (i8 -42)
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD1]], splat (i8 42)
; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
@@ -201,8 +199,6 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP15]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP19:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
@@ -211,16 +207,18 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP18]], <4 x i1> [[TMP22]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP23]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP26]], i64 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP26]], align 1
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD7]]
-; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP29]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP15]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP18]], <4 x i1> [[TMP22]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP23]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> zeroinitializer, <4 x i1> [[TMP28]]
+; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> zeroinitializer, <4 x i1> [[TMP29]]
; CHECK-NEXT: [[TMP32:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP24]]
; CHECK-NEXT: [[TMP33:%.*]] = select <4 x i1> [[TMP17]], <4 x i1> splat (i1 true), <4 x i1> [[TMP25]]
; CHECK-NEXT: [[TMP34:%.*]] = select <4 x i1> [[TMP32]], <4 x i1> splat (i1 true), <4 x i1> [[TMP30]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll b/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
index f672f649915a4..8110b215ad637 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
@@ -17,15 +17,14 @@ define i64 @diamond_with_2_early_exits() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i64>
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> zeroinitializer, <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
@@ -94,24 +93,23 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[CHECK_B:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> zeroinitializer, <4 x i1> [[TMP14]]
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP15]]
; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP17]]
@@ -193,22 +191,20 @@ define i64 @nested_diamond_inner_exits() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> zeroinitializer, <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> zeroinitializer, <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[PREDPHI]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP15]]
@@ -297,15 +293,15 @@ define i64 @chain_of_3_exits() {
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_C]], align 1
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP8]], align 1
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> splat (i1 true), <4 x i1> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> splat (i1 true), <4 x i1> [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP12]]
@@ -383,22 +379,20 @@ define i64 @four_exits_2x2_diamond() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> zeroinitializer, <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD3]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
; CHECK-NEXT: [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> zeroinitializer, <4 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP13]]
@@ -577,10 +571,9 @@ define i64 @diamond_exits_overlapping_conditions() {
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_C]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> zeroinitializer, <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
>From e181c560cb93f735f8d0a3ca41a7a79b610d8ba6 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 16 Sep 2026 22:36:23 +0800
Subject: [PATCH 3/4] Add assert there is a defining recipe
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a1b4aebf9fd7e..03065b7bc906c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3487,6 +3487,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(
// Add phis so IncomingVal is defined on all paths to the latch.
DenseMap<VPBasicBlock *, VPValue *> Defs = {
{HeaderVPBB, Plan.getPoison(IncomingVal->getScalarType())}};
+ assert(IncomingVal->hasDefiningRecipe() &&
+ "Non-live-in IncomingVal without a recipe?");
VPBasicBlock *DefVPBB = IncomingVal->getDefiningRecipe()->getParent();
assert(VPDT.dominates(HeaderVPBB, DefVPBB) &&
"IncomingVal defined outside of vector body?");
>From fbb4b0ec1a0102f7c201821657ea5f1bacc5f329 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 21 Sep 2026 13:47:47 +0100
Subject: [PATCH 4/4] Only assign to defs when headervpbb != colliding block
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 03065b7bc906c..e5aa17d926555 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3340,10 +3340,11 @@ bool VPlanTransforms::handleUncountableEarlyExits(
// Add phis so there's a def of CondToEarlyExit on every path leading to the
// latch. The condition is false on paths that didn't go through
- // EarlyExitingVPBB. EarlyExitingVPBB may be the same as HeaderVPBB, so
- // assign in order.
- DenseMap<VPBasicBlock *, VPValue *> Defs = {{HeaderVPBB, Plan.getFalse()}};
- Defs[EarlyExitingVPBB] = CondToEarlyExit;
+ // EarlyExitingVPBB.
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {EarlyExitingVPBB, CondToEarlyExit}};
+ if (HeaderVPBB != EarlyExitingVPBB)
+ Defs[HeaderVPBB] = Plan.getFalse();
CondToEarlyExit = vputils::reconstructSSA(LatchVPBB, Defs);
Exits.push_back({
@@ -3485,14 +3486,14 @@ bool VPlanTransforms::handleUncountableEarlyExits(
VPValue *NewIncoming = IncomingVal;
if (!isa<VPIRValue>(IncomingVal)) {
// Add phis so IncomingVal is defined on all paths to the latch.
- DenseMap<VPBasicBlock *, VPValue *> Defs = {
- {HeaderVPBB, Plan.getPoison(IncomingVal->getScalarType())}};
assert(IncomingVal->hasDefiningRecipe() &&
"Non-live-in IncomingVal without a recipe?");
VPBasicBlock *DefVPBB = IncomingVal->getDefiningRecipe()->getParent();
assert(VPDT.dominates(HeaderVPBB, DefVPBB) &&
"IncomingVal defined outside of vector body?");
- Defs[DefVPBB] = IncomingVal;
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{DefVPBB, IncomingVal}};
+ if (HeaderVPBB != DefVPBB)
+ Defs[HeaderVPBB] = Plan.getPoison(IncomingVal->getScalarType());
IncomingVal = vputils::reconstructSSA(LatchVPBB, Defs);
VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
More information about the llvm-commits
mailing list