[llvm] [VPlan] Create AnyOf and FindIV resume adjustments in the epilogue plan. (PR #227882)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 12:34:48 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/227882
>From cd828b8c3af39a6e65198ebe62c194784307f1e7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 27 Sep 2026 11:16:40 +0100
Subject: [PATCH 1/3] [VPlan] Create AnyOf and FindIV resume adjustments in the
epilogue plan.
AnyOf and FindIV reductions adjust the resume value from the main vector loop
before using it as start value of the epilogue vector loop. The adjustments
were created as IR instructions next to the resume phi generated by the main
plan, and moved to the preheader of the epilogue vector loop together with the
phi after executing the epilogue plan.
Create them as VPInstructions in the epilogue plan's vector preheader instead.
They are broadcast explicitly, as values computed in the vector preheader are
not covered by materializeBroadcasts and would otherwise be widened. This
removes the instructions to move from connectEpilogueVectorLoop.
Test changes are the order of the instructions in the epilogue vector loop's
preheader, and a now folded compare of an i1 AnyOf resume value against false.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 66 ++++++++-----------
.../AArch64/epilog-iv-select-cmp.ll | 12 ++--
...-vectorization-fix-scalar-resume-values.ll | 5 +-
.../find-last-iv-sinkable-expr-epilogue.ll | 4 +-
.../LoopVectorize/AArch64/select-index.ll | 6 +-
.../VPlan/epilogue-vectorization-printing.ll | 11 ++--
.../LoopVectorize/epilog-iv-select-cmp.ll | 14 ++--
.../epilog-vectorization-any-of-reductions.ll | 15 ++---
...ilog-vectorization-function-entry-count.ll | 6 +-
.../epilog-vectorization-reductions.ll | 6 +-
.../select-min-max-last-index-epilogue.ll | 16 ++---
11 files changed, 77 insertions(+), 84 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45edf702d7092..55c2c37c54a7e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7396,13 +7396,15 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
}));
}
+/// Returns the value the executed main VPlan generated for \p MainV as a
+/// live-in of \p EpiPlan.
+static VPValue *getMainPlanValueAsLiveIn(VPlan &EpiPlan, const VPValue *MainV) {
+ return EpiPlan.getOrAddLiveIn(MainV->getUnderlyingValue());
+}
+
/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
-/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
-/// reductions require creating new instructions to compute the resume values.
-/// They are collected in a vector and returned. They must be moved to the
-/// preheader of the vector epilogue loop, after created by the execution of \p
-/// Plan.
-static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
+/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
+static void preparePlanForEpilogueVectorLoop(
VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
EpilogueLoopVectorizationInfo &EPI, LoopVectorizationPlanner &LVP,
VFSelectionContext &Config, ScalarEvolution &SE,
@@ -7471,7 +7473,6 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
OffsetIVInc->setOperand(0, Increment);
DenseMap<Value *, Value *> ToFrozen;
- SmallVector<Instruction *> InstsToMove;
// Ensure that the start values for all header phi recipes are updated before
// vectorizing the epilogue loop.
for (VPRecipeBase &R : Header->phis()) {
@@ -7504,38 +7505,38 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) || IsFindIV) {
- auto *ResumePhi = cast<PHINode>(ResumeV);
VPValue *BypassOp = ResumeForEpi->getOperand(1);
assert((isa<VPIRValue>(BypassOp) ||
VPlanPatternMatch::match(
BypassOp,
m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
"expected live-in or Freeze");
- Value *StartV = BypassOp->getUnderlyingValue();
- IRBuilder<> Builder(ResumePhi->getParent(),
- ResumePhi->getParent()->getFirstNonPHIIt());
+ VPValue *StartV = getMainPlanValueAsLiveIn(Plan, BypassOp);
+ VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
+ VPBasicBlock *VectorPH = Plan.getVectorPreheader();
+ VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
- ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
- if (auto *I = dyn_cast<Instruction>(ResumeV))
- InstsToMove.push_back(I);
+ StartVal = PHBuilder.createICmp(CmpInst::ICMP_NE, StartVal, StartV);
} else {
assert(SentinelVPV && "expected to find icmp using RdxResult");
- if (auto *FreezeI = dyn_cast<FreezeInst>(StartV))
- ToFrozen[FreezeI->getOperand(0)] = StartV;
+ assert(!SentinelVPV->getDefiningRecipe() &&
+ "sentinel must be a live-in to be used in the preheader");
+ if (auto *FreezeI = dyn_cast<FreezeInst>(StartV->getLiveInIRValue()))
+ ToFrozen[FreezeI->getOperand(0)] = FreezeI;
// Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
- Value *Cmp = Builder.CreateICmpEQ(ResumeV, StartV);
- if (auto *I = dyn_cast<Instruction>(Cmp))
- InstsToMove.push_back(I);
- ResumeV = Builder.CreateSelect(Cmp, SentinelVPV->getLiveInIRValue(),
- ResumeV);
- if (auto *I = dyn_cast<Instruction>(ResumeV))
- InstsToMove.push_back(I);
+ VPValue *Cmp =
+ PHBuilder.createICmp(CmpInst::ICMP_EQ, StartVal, StartV);
+ StartVal = PHBuilder.createSelect(Cmp, SentinelVPV, StartVal);
}
+ // materializeBroadcasts does not cover values in the vector preheader.
+ ReductionPhi->setStartValue(
+ PHBuilder.createNaryOp(VPInstruction::Broadcast, {StartVal}));
+ continue;
} else {
VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
@@ -7622,8 +7623,6 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
EPI.VectorTripCount, Plan.requiresScalarEpilogue(),
EPI.EpilogueVF, MainLoopStep, EpilogueLoopStep, SE);
-
- return InstsToMove;
}
static void
@@ -7651,12 +7650,9 @@ fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, VPlan &BestEpiPlan,
/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
/// loop, after both plans have executed, updating the branch from the iteration
-/// count check of the main loop, as well as updating various phis. \p
-/// InstsToMove contains instructions that need to be moved to the preheader of
-/// the epilogue vector loop.
+/// count check of the main loop, as well as updating various phis.
static void connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
VPIRBasicBlock *VecEpilogueIterCheckVPBB,
- ArrayRef<Instruction *> InstsToMove,
ArrayRef<VPInstruction *> ResumeValues) {
ArrayRef<VPBlockBase *> Preds = VecEpilogueIterCheckVPBB->getPredecessors();
BasicBlock *MainLoopIterationCountCheck =
@@ -7689,10 +7685,6 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
VecEpilogueIterationCountCheck);
}
- auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
- for (auto *I : InstsToMove)
- I->moveBefore(IP);
-
// VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
// after executing the main loop. We need to update the resume values of
// inductions and reductions during epilogue vectorization.
@@ -8178,16 +8170,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// edges from the first pass.
EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI,
Checks, BestEpiPlan, BestMainPlan);
- SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
- BestMainPlan, BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
- *PSE.getSE(), ResumeValues);
+ preparePlanForEpilogueVectorLoop(BestMainPlan, BestEpiPlan, L,
+ ExpandedSCEVs, EPI, LVP, Config,
+ *PSE.getSE(), ResumeValues);
RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
LVP.executePlan(
EPI.EpilogueVF, /*BestUF=*/1, BestEpiPlan, EpilogILV, DT,
LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
connectEpilogueVectorLoop(BestEpiPlan, DT,
EpilogILV.VecEpilogueIterationCountCheck,
- InstsToMove, ResumeValues);
+ ResumeValues);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
index 84d0297fcb9a3..2e858bdcaddc5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
@@ -52,13 +52,13 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i8 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[FR]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i8 [[BC_MERGE_RDX]], [[FR]]
-; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i8 -128, i8 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = and i32 [[TMP2]], 7
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF4]]
-; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[N_VEC5]] to i8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i8 [[BC_MERGE_RDX]], [[FR]]
+; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i8 -128, i8 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP15]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i8> [[DOTSPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[N_VEC5]] to i8
; CHECK-NEXT: [[DOTSPLATINSERT10:%.*]] = insertelement <8 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
; CHECK-NEXT: [[DOTSPLAT11:%.*]] = shufflevector <8 x i8> [[DOTSPLATINSERT10]], <8 x i8> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <8 x i8> [[DOTSPLAT11]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
@@ -170,13 +170,13 @@ define i32 @select_icmp_var_start_iv_trunc(i32 %N, i32 %start) #0 {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[FR]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], [[FR]]
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 -2147483648, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF7:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC8:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF7]]
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[START]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], [[FR]]
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 -2147483648, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[DOTSPLATINSERT13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
; CHECK-NEXT: [[DOTSPLAT14:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT13]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[START]], 0
; CHECK-NEXT: [[TMP12:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
index 16d8e5751e289..4c0b9e4695227 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
@@ -220,13 +220,12 @@ define i64 @find_last_and_any_of(ptr %Dst, ptr %Src) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 992, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[TMP20]], %[[VEC_EPILOG_ITER_CHECK]] ], [ false, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i1> poison, i1 [[BC_MERGE_RDX7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT8]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], -1
; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP21]], i32 -2147483648, i32 [[BC_MERGE_RDX]]
-; CHECK-NEXT: [[TMP23:%.*]] = icmp ne i1 [[BC_MERGE_RDX7]], false
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP22]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i1> poison, i1 [[TMP23]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT8]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP24:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT10]], <4 x i32> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll
index d9da6f751a872..9065912d738c7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll
@@ -65,10 +65,10 @@ define i32 @findlast_add_expr(ptr %a, i32 %n) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], -1
-; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], i32 0, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF10:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i32 [[N]], [[N_MOD_VF10]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], -1
+; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], i32 0, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP21]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <4 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
index 0e3591c57b070..d887ccabb9981 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll
@@ -1031,13 +1031,13 @@ define i32 @test_multi_use_reduction_with_trunc_iv(ptr %src, i32 %n) {
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], 0
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP31]], i32 0, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF18:%.*]] = and i64 [[N_EXT]], 3
; CHECK-NEXT: [[N_VEC19:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF18]]
-; CHECK-NEXT: [[TMP33:%.*]] = add i64 1, [[N_VEC19]]
+; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP31]], i32 0, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[TMP32]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT21:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT20]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = add i64 1, [[N_VEC19]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[BC_MERGE_RDX17]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT23:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT22]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP34:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
index e477b32265026..7e9dd7640efd6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll
@@ -938,12 +938,13 @@ define i32 @any_of_resume(ptr noalias %src, i64 %n) {
; CHECK: vec.epilog.ph:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = and ir<%n>, ir<3>
; CHECK-NEXT: EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast ir<[[ADJ:%.+]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%bc.merge.rdx>, ir<0>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = broadcast vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vec.epilog.vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vec.epilog.vector.body:
; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<%vec.epilog.resume.val>, vec.epilog.ph ], [ vp<%index.next>, vec.epilog.vector.body ]
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (any-of) vp<[[VP4]]>, vp<{{.+}}>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (any-of) vp<[[VP5]]>, vp<{{.+}}>
;
entry:
br label %loop
@@ -969,9 +970,11 @@ define i64 @find_iv_resume(ptr noalias %a, i64 %n) {
; CHECK: vec.epilog.ph:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = and ir<%n>, ir<3>
; CHECK-NEXT: EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast ir<[[ADJ:%.+]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp eq ir<%bc.merge.rdx>, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = select vp<[[VP4]]>, ir<-9223372036854775808>, ir<%bc.merge.rdx>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = broadcast vp<[[VP5]]>
; CHECK: vec.epilog.vector.body:
-; CHECK: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) vp<[[VP4]]>, ir<%sel>
+; CHECK: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) vp<[[VP6]]>, ir<%sel>
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
index aacfbeaf74483..d41bd3b8a50fe 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
@@ -38,10 +38,10 @@ define i64 @select_icmp_const(ptr %a, i64 %n) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 3
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 3
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
@@ -139,10 +139,10 @@ define i64 @select_fcmp_const_fast(ptr %a, i64 %n) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 2
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
@@ -247,13 +247,13 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[FR]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i8 [[BC_MERGE_RDX]], [[FR]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i8 -128, i8 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i32 [[TMP2]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF2]]
-; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[N_VEC3]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i8 [[BC_MERGE_RDX]], [[FR]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i8 -128, i8 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP12]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[N_VEC3]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i8> poison, i8 [[BC_RESUME_VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT4]], <4 x i8> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i8> [[BROADCAST_SPLAT5]], <i8 0, i8 1, i8 2, i8 3>
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
index 8d237eb9386f9..6603ee0cf5eb6 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
@@ -39,9 +39,9 @@ define i32 @any_of_reduction_epilog(ptr %src, i64 %N) {
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
; CHECK-NEXT: [[MINMAX_IDENT_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP9]], i64 0
; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[MINMAX_IDENT_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -133,9 +133,9 @@ define i32 @any_of_reduction_epilog_arg_as_start_value(ptr %src, i64 %N, i32 %st
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], [[START]]
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], [[START]]
; CHECK-NEXT: [[MINMAX_IDENT_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP9]], i64 0
; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[MINMAX_IDENT_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -230,14 +230,13 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], false
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[IND_END5:%.*]] = trunc i64 [[N_VEC3]] to i32
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
-; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[BC_RESUME_VAL]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3>
@@ -380,15 +379,15 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ true, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP48:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], true
; CHECK-NEXT: [[N_MOD_VF7:%.*]] = and i64 [[TMP3]], 3
; CHECK-NEXT: [[N_VEC8:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF7]]
+; CHECK-NEXT: [[TMP48:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], true
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i1> poison, i1 [[TMP48]], i64 0
+; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT13]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP25:%.*]] = shl i64 [[N_VEC8]], 4
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP25]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT19:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT18]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i1> poison, i1 [[TMP48]], i64 0
-; CHECK-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT13]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-function-entry-count.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-function-entry-count.ll
index 24bdf7d32f527..61d28ab48e3f1 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-function-entry-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-function-entry-count.ll
@@ -197,9 +197,9 @@ define i32 @any_of_reduction(ptr %src, i64 %n) !prof !0 {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 3
; CHECK-NEXT: [[TMP8:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC2:%.*]] = sub i64 [[N]], [[TMP8]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 3
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP7]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -292,10 +292,10 @@ define i64 @find_last_iv(ptr %src, i64 %n) !prof !0 {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], -1
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[TMP10:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC2:%.*]] = sub i64 [[N]], [[TMP10]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], -1
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP18]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index a662ac9967adb..c2101493aa563 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -944,15 +944,15 @@ define i32 @anyof_reduction_in_dissolved_epilogue(i32 %val, i1 %c) {
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
; CHECK-NEXT: [[N_MOD_VF1:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[N_VEC2:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF1]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[START]], [[N_VEC2]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne i32 [[VAL]], 0
; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[TMP9:%.*]] = or <4 x i1> [[BROADCAST_SPLAT4]], [[BROADCAST_SPLAT6]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll b/llvm/test/Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll
index 261feb4ce1c2a..5e68eda8211b9 100644
--- a/llvm/test/Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll
@@ -46,10 +46,10 @@ define i64 @test_vectorize_select_umax_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP8]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP11]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP11]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX3]], i64 0
@@ -170,10 +170,10 @@ define i64 @test_vectorize_select_umax_idx_cond_flipped(ptr %src, i64 %n) {
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP8]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP11]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[N_MOD_VF4:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[BC_MERGE_RDX]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP11]], i64 -9223372036854775808, i64 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[BC_MERGE_RDX3]], i64 0
@@ -295,16 +295,16 @@ define i32 @test_vectorize_select_smax_idx(i32 %n, i32 %val) {
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i32 [ [[RDX_SELECT5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BC_MERGE_RDX6]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 -2147483648, i32 [[BC_MERGE_RDX6]]
; CHECK-NEXT: [[N_MOD_VF7:%.*]] = and i32 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC8:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF7]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BC_MERGE_RDX6]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 -2147483648, i32 [[BC_MERGE_RDX6]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT13]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i32> poison, i32 [[VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT9]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i32> poison, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT11]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT13]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i32> poison, i32 [[BC_RESUME_VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT15]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i32> [[BROADCAST_SPLAT16]], <i32 0, i32 1, i32 2, i32 3>
>From e61c38482b3f241258989f88a82e24ca2a97ce21 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 2 Oct 2026 15:51:46 +0100
Subject: [PATCH 2/3] !fixup address comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 69 +++++++++----------
...-vectorization-fix-scalar-resume-values.ll | 12 ++--
2 files changed, 38 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 339aa9e8701a2..77791bd67534c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7266,9 +7266,8 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
VPInstruction *Freeze = Builder.createFreeze(OrigStart, {}, "fr");
VPI.setOperand(2, Freeze);
if (UpdateResumePhis)
- OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
- return Freeze != &U && isa<VPPhi>(&U);
- });
+ OrigStart->replaceUsesWithIf(
+ Freeze, [](VPUser &U, unsigned) { return isa<VPPhi>(&U); });
}
};
AddFreezeForFindLastIVReductions(MainPlan, true);
@@ -7326,12 +7325,6 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
}));
}
-/// Returns the value the executed main VPlan generated for \p MainV as a
-/// live-in of \p EpiPlan.
-static VPValue *getMainPlanValueAsLiveIn(VPlan &EpiPlan, const VPValue *MainV) {
- return EpiPlan.getOrAddLiveIn(MainV->getUnderlyingValue());
-}
-
/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
static void preparePlanForEpilogueVectorLoop(
@@ -7402,11 +7395,16 @@ static void preparePlanForEpilogueVectorLoop(
Increment->replaceAllUsesWith(OffsetIVInc);
OffsetIVInc->setOperand(0, Increment);
- DenseMap<Value *, Value *> ToFrozen;
+ DenseMap<VPValue *, VPValue *> ToFrozen;
+
+ // Resume values must be created in the vector preheader.
+ VPBasicBlock *VectorPH = Plan.getVectorPreheader();
+ VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
+
// Ensure that the start values for all header phi recipes are updated before
// vectorizing the epilogue loop.
for (VPRecipeBase &R : Header->phis()) {
- Value *ResumeV = nullptr;
+ VPValue *ResumeVPV = nullptr;
// TODO: Move setting of resume values to prepareToExecute.
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
// Find the reduction result by searching users of the phi or its backedge
@@ -7421,7 +7419,6 @@ static void preparePlanForEpilogueVectorLoop(
VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
cast<PHINode>(ReductionPhi->getUnderlyingInstr()));
- ResumeV = ResumeForEpi->getUnderlyingValue();
// Check for FindIV pattern by looking for icmp user of RdxResult.
// The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
@@ -7434,6 +7431,7 @@ static void preparePlanForEpilogueVectorLoop(
});
RecurKind RK = ReductionPhi->getRecurrenceKind();
+ ResumeVPV = Plan.getOrAddLiveIn(ResumeForEpi->getUnderlyingValue());
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) || IsFindIV) {
VPValue *BypassOp = ResumeForEpi->getOperand(1);
assert((isa<VPIRValue>(BypassOp) ||
@@ -7441,34 +7439,32 @@ static void preparePlanForEpilogueVectorLoop(
BypassOp,
m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
"expected live-in or Freeze");
- VPValue *StartV = getMainPlanValueAsLiveIn(Plan, BypassOp);
- VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
- VPBasicBlock *VectorPH = Plan.getVectorPreheader();
- VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
-
+ VPValue *StartV = Plan.getOrAddLiveIn(BypassOp->getUnderlyingValue());
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
- StartVal = PHBuilder.createICmp(CmpInst::ICMP_NE, StartVal, StartV);
+ ResumeVPV = PHBuilder.createICmp(CmpInst::ICMP_NE, ResumeVPV, StartV);
} else {
- assert(SentinelVPV && "expected to find icmp using RdxResult");
- assert(!SentinelVPV->getDefiningRecipe() &&
+ assert(isa<VPIRValue>(SentinelVPV) &&
"sentinel must be a live-in to be used in the preheader");
- if (auto *FreezeI = dyn_cast<FreezeInst>(StartV->getLiveInIRValue()))
- ToFrozen[FreezeI->getOperand(0)] = FreezeI;
+ VPValue *OrigStart;
+ if (VPlanPatternMatch::match(
+ BypassOp, VPlanPatternMatch::m_Freeze(m_VPValue(OrigStart))))
+ ToFrozen[Plan.getOrAddLiveIn(cast<VPIRValue>(OrigStart))] = StartV;
- // Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
+ // Adjust resume: select(icmp eq ResumeVPV, StartV), Sentinel,
+ // ResumeVPV
VPValue *Cmp =
- PHBuilder.createICmp(CmpInst::ICMP_EQ, StartVal, StartV);
- StartVal = PHBuilder.createSelect(Cmp, SentinelVPV, StartVal);
+ PHBuilder.createICmp(CmpInst::ICMP_EQ, ResumeVPV, StartV);
+ ResumeVPV = PHBuilder.createSelect(Cmp, SentinelVPV, ResumeVPV);
}
- // materializeBroadcasts does not cover values in the vector preheader.
+ // TODO: materializeBroadcasts does not cover values in the vector
+ // preheader.
ReductionPhi->setStartValue(
- PHBuilder.createNaryOp(VPInstruction::Broadcast, {StartVal}));
+ PHBuilder.createNaryOp(VPInstruction::Broadcast, {ResumeVPV}));
continue;
} else {
- VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
@@ -7490,7 +7486,7 @@ static void preparePlanForEpilogueVectorLoop(
// For FP sub-reductions, verify start value is negative zero.
[[maybe_unused]] auto StartValueIsIdentity = [&] {
Value *IdentityValue = getRecurrenceIdentity(
- PhiR->getRecurrenceKind(), ResumeV->getType(),
+ PhiR->getRecurrenceKind(), ResumeVPV->getScalarType(),
PhiR->getFastMathFlagsOrNone());
auto *StartValue = dyn_cast<VPIRValue>(VPI->getOperand(0));
return StartValue && StartValue->getValue() == IdentityValue;
@@ -7499,20 +7495,20 @@ static void preparePlanForEpilogueVectorLoop(
"Expected start value for partial sub-reduction to be zero "
"(or negative zero)");
- Sub->setOperand(0, StartVal);
+ Sub->setOperand(0, ResumeVPV);
} else
- VPI->setOperand(0, StartVal);
+ VPI->setOperand(0, ResumeVPV);
continue;
}
}
} else {
// Retrieve the induction resume value via ResumeForEpilogue.
PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
- ResumeV = IRPhiToResumeForEpi.at(IndPhi)->getUnderlyingValue();
+ ResumeVPV = Plan.getOrAddLiveIn(
+ IRPhiToResumeForEpi.at(IndPhi)->getUnderlyingValue());
}
- assert(ResumeV && "Must have a resume value");
- VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
- cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
+ assert(ResumeVPV && "Must have a resume value");
+ cast<VPHeaderPHIRecipe>(&R)->setStartValue(ResumeVPV);
}
// For some VPValues in the epilogue plan we must re-use the generated IR
@@ -7525,8 +7521,7 @@ static void preparePlanForEpilogueVectorLoop(
// epilogue plan. This ensures all users use the same frozen value.
auto *VPI = dyn_cast<VPInstruction>(&R);
if (VPI && VPI->getOpcode() == Instruction::Freeze) {
- VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
- ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
+ VPI->replaceAllUsesWith(ToFrozen.lookup(VPI->getOperand(0)));
continue;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
index 4c0b9e4695227..d148a7907f9d4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
@@ -123,12 +123,12 @@ define void @epilogue_main_loop_dead(ptr %dst, i32 %width, ptr %src) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL11:%.*]] = phi ptr [ [[TMP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[DST]], %[[VECTOR_MEMCHECK1]] ], [ [[DST]], %[[LOOP_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL12:%.*]] = phi i32 [ [[TMP8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[WIDTH]], %[[VECTOR_MEMCHECK1]] ], [ [[WIDTH]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL10:%.*]] = phi ptr [ [[TMP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[DST]], %[[VECTOR_MEMCHECK1]] ], [ [[DST]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL11:%.*]] = phi i32 [ [[TMP8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[WIDTH]], %[[VECTOR_MEMCHECK1]] ], [ [[WIDTH]], %[[LOOP_PREHEADER]] ]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[D:%.*]] = phi ptr [ [[D_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL11]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[W:%.*]] = phi i32 [ [[W_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL12]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[D:%.*]] = phi ptr [ [[D_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL10]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[W:%.*]] = phi i32 [ [[W_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL11]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
; CHECK-NEXT: store i8 [[V]], ptr [[D]], align 1
; CHECK-NEXT: [[D_NEXT]] = getelementptr i8, ptr [[D]], i64 1
@@ -220,12 +220,12 @@ define i64 @find_last_and_any_of(ptr %Dst, ptr %Src) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 992, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[TMP20]], %[[VEC_EPILOG_ITER_CHECK]] ], [ false, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i1> poison, i1 [[BC_MERGE_RDX7]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT8]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], -1
; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP21]], i32 -2147483648, i32 [[BC_MERGE_RDX]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP22]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i1> poison, i1 [[BC_MERGE_RDX7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT8]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP24:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT10]], <4 x i32> poison, <4 x i32> zeroinitializer
>From 39e2242357461a26ee55588f6aa2226a0d49bd01 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 3 Oct 2026 20:34:02 +0100
Subject: [PATCH 3/3] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1ff4885b4adc0..521f6d15a8c18 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7434,7 +7434,7 @@ static void preparePlanForEpilogueVectorLoop(
Increment->replaceAllUsesWith(OffsetIVInc);
OffsetIVInc->setOperand(0, Increment);
- DenseMap<VPValue *, VPValue *> ToFrozen;
+ SmallDenseMap<VPValue *, VPValue *, 8> ToFrozen;
// Resume values must be created in the vector preheader.
VPBasicBlock *VectorPH = Plan.getVectorPreheader();
@@ -7501,7 +7501,7 @@ static void preparePlanForEpilogueVectorLoop(
// TODO: materializeBroadcasts does not cover values in the vector
// preheader.
ReductionPhi->setStartValue(
- PHBuilder.createNaryOp(VPInstruction::Broadcast, {ResumeVPV}));
+ PHBuilder.createNaryOp(VPInstruction::Broadcast, ResumeVPV));
continue;
} else {
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
More information about the llvm-commits
mailing list