[llvm] 622e816 - [VPlan] Fix VPInstruction::AnyOf combine undoing freeze (#223640)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 02:22:12 PDT 2026
Author: Luke Lau
Date: 2026-09-21T10:22:05+01:00
New Revision: 622e816cb4bf94ea99e2f1763c5c16b061ec80e2
URL: https://github.com/llvm/llvm-project/commit/622e816cb4bf94ea99e2f1763c5c16b061ec80e2
DIFF: https://github.com/llvm/llvm-project/commit/622e816cb4bf94ea99e2f1763c5c16b061ec80e2.diff
LOG: [VPlan] Fix VPInstruction::AnyOf combine undoing freeze (#223640)
There is an any-of combine for unrolled VPlans which does:
any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
However any-of implicitly freezes each individual operand and this means
we go from `freeze (fcmp uno A, A)` to `freeze (fcmp uno A, B)` which
isn't sound: https://alive2.llvm.org/ce/z/UdQM7C
This causes miscompiles today with early exit loops that are unrolled,
see the attached test case in single-early-exit-anyof-fold.ll.
This fixes it by explicitly modelling the freeze in VPlan. There are
three places where we use AnyOf:
1) early exit loops: the freeze needs to be applied per-lane, so apply
it as `(any-of (freeze (combined-conds-to-exit)))`
2) handleMaxMinNumReductions: we need to freeze the operands to the
min/max reductions earlier since they have multiple uses introduced and
we don't check the values aren't undef
3) handleFindLastReductions: we need to freeze Cond itself since it's
got multiple uses, but I plan on fixing this in a separate PR
Fable 5.1 was used to help generate the tests
Added:
llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
Modified:
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
llvm/test/Transforms/LoopVectorize/select-cmp.ll
llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index daabca902724b..4534ff881fcad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1378,8 +1378,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Returns a scalar boolean value, which is true if any lane of its
// (boolean) vector operands is true. It produces the reduced value across
// all unrolled iterations. Unrolling will add all copies of its original
- // operand as additional operands. AnyOf is poison-safe as all operands
- // will be frozen.
+ // operand as additional operands. Note does not block poison propagation.
AnyOf,
// Calculates the first active lane index of the vector predicate operands.
// It produces the lane index across all unrolled iterations. Unrolling will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 9eb7f7b1223f6..18498084a601a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1695,6 +1695,17 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
}
}
+ // Freeze MinOrMaxOps since:
+ // * multiple uses are introduced and the value may be undef
+ // * they feed into a branch condition, so block poison propagation to
+ // prevent immediate UB
+ for (auto &[Phi, MinOrMaxOp] : MinOrMaxNumReductionsToHandle) {
+ VPRecipeBase *MinOrMaxR = Phi->getBackedgeValue()->getDefiningRecipe();
+ VPInstruction *Freeze = VPBuilder(MinOrMaxR).createFreeze(MinOrMaxOp);
+ MinOrMaxR->replaceUsesOfWith(MinOrMaxOp, Freeze);
+ MinOrMaxOp = Freeze;
+ }
+
VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
VPValue *AllNaNLanes = nullptr;
@@ -1885,7 +1896,9 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
if (HeaderMask)
Cond = Builder.createLogicalAnd(HeaderMask, Cond);
- VPValue *AnyOf = Builder.createNaryOp(VPInstruction::AnyOf, {Cond});
+ VPValue *AnyOf =
+ Builder.createNaryOp(VPInstruction::AnyOf, Builder.createFreeze(Cond));
+ // FIXME: The Cond here needs to be frozen too.
VPValue *MaskSelect = Builder.createSelect(AnyOf, Cond, MaskPHI);
MaskPHI->addIncoming(MaskSelect);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 452589bafd534..602aff8090bf8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1036,9 +1036,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::AnyOf: {
- Value *Res = Builder.CreateFreeze(State.get(getOperand(0)));
+ Value *Res = State.get(getOperand(0));
for (VPValue *Op : drop_begin(operands()))
- Res = Builder.CreateOr(Res, Builder.CreateFreeze(State.get(Op)));
+ Res = Builder.CreateOr(Res, State.get(Op));
return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
}
case VPInstruction::ExtractLane: {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 64099a65f5c74..a2f2a2997086e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3047,7 +3047,8 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
// EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
// EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
- // EMIT vp<%4> = any-of ir<%3>
+ // EMIT vp<%freeze> = freeze ir<%3>
+ // EMIT vp<%4> = any-of ir<%freeze>
// EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
// EMIT branch-on-two-conds vp<%4>, vp<%5>
// Successor(s): middle.block, middle.block, for.body
@@ -3100,10 +3101,12 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
return nullptr;
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
Recipes.push_back(cast<VPInstruction>(GepR));
- } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op1)))) {
- Worklist.push_back(Op1);
+ } else if (match(V, m_Freeze(m_VPValue(
+ Op1, m_VPInstruction<VPInstruction::MaskedCond>(
+ m_VPValue(Op2)))))) {
+ Worklist.push_back(Op2);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ Recipes.push_back(cast<VPInstruction>(Op1->getDefiningRecipe()));
} else
return nullptr;
}
@@ -3144,7 +3147,8 @@ struct EarlyExitInfo {
/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
/// EMIT store ir<%add>, ir<%arrayidx5>
/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
-/// EMIT vp<%3> = any-of ir<%1>
+/// EMIT vp<%freeze> = freeze ir<%1>
+/// EMIT vp<%3> = any-of ir<%freeze>
/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
/// Successor(s): middle.block, middle.block, for.body
@@ -3384,8 +3388,15 @@ bool VPlanTransforms::handleUncountableEarlyExits(
for (const EarlyExitInfo &Info : drop_begin(Exits))
Combined = LatchBuilder.createLogicalOr(Combined, Info.CondToExit);
- VPValue *IsAnyExitTaken =
- LatchBuilder.createNaryOp(VPInstruction::AnyOf, {Combined});
+ // Even though the logical or prevents posion propagation, we need to freeze
+ // Combined to prevent poisoning the entire AnyOf result:
+ //
+ // Exits[0].CondToExit = [0,1,0,0]
+ // Exits[1].CondToExit = [0,0,p,p]
+ // Combined = [0,1,p,p]
+ // AnyOf = 1
+ VPValue *IsAnyExitTaken = LatchBuilder.createNaryOp(
+ VPInstruction::AnyOf, LatchBuilder.createFreeze(Combined));
// Create a comparison for the latch exit condition and replace the
// BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index a25edccb60514..7508461d4e4ec 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -15,17 +15,18 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = any-of vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i32 %iv to float
@@ -33,29 +34,30 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i32 %iv.next, %n
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 4: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP4]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP4]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP5]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP6]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP7]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP5]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP6]]> = any-of vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP8]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %iv.f = sitofp i32 %iv to float
@@ -63,14 +65,14 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 4: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 4: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 4: IR %ec = icmp eq i32 %iv.next, %n
-; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP9]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP13]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -96,17 +98,18 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to double
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = any-of vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi double [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i64 %iv to double
@@ -114,14 +117,14 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 0 for VF 2: IR %max.next = call double @llvm.maxnum.f64(double %max, double %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i64 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: LV: Selecting VF: 1.
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 38c8133ddc8d1..faba88cee2203 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -66,13 +66,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -344,14 +344,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -468,13 +468,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR1]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1140,13 +1140,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 09ccca93fa368..7aff41be08a00 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -62,15 +62,17 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP25]])
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP26]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP27]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP22]], [[TMP25]]
+; CHECK-NEXT: [[TMP23:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP27]]
; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -161,25 +163,29 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP22]], align 4
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP28]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP10]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI6]], <4 x float> [[WIDE_LOAD9]])
-; CHECK-NEXT: [[TMP11]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI7]], <4 x float> [[WIDE_LOAD10]])
-; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
-; CHECK-NEXT: [[TMP14]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[WIDE_LOAD13]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI5]], <4 x float> [[WIDE_LOAD14]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP29:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP34:%.*]] = freeze <4 x float> [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP43:%.*]] = freeze <4 x float> [[WIDE_LOAD10]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP29]])
+; CHECK-NEXT: [[TMP10]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI6]], <4 x float> [[TMP34]])
+; CHECK-NEXT: [[TMP11]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI7]], <4 x float> [[TMP43]])
+; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP45:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP46:%.*]] = freeze <4 x float> [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP47:%.*]] = freeze <4 x float> [[WIDE_LOAD14]]
+; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP44]])
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP45]])
+; CHECK-NEXT: [[TMP14]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[TMP46]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI5]], <4 x float> [[TMP47]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP29:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD13]], [[WIDE_LOAD9]]
-; CHECK-NEXT: [[TMP34:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD14]], [[WIDE_LOAD10]]
-; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp uno <4 x float> [[TMP44]], [[TMP9]]
+; CHECK-NEXT: [[TMP17:%.*]] = fcmp uno <4 x float> [[TMP45]], [[TMP29]]
+; CHECK-NEXT: [[TMP35:%.*]] = fcmp uno <4 x float> [[TMP46]], [[TMP34]]
+; CHECK-NEXT: [[TMP41:%.*]] = fcmp uno <4 x float> [[TMP47]], [[TMP43]]
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP35:%.*]] = freeze <4 x i1> [[TMP29]]
; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP18]], [[TMP35]]
-; CHECK-NEXT: [[TMP41:%.*]] = freeze <4 x i1> [[TMP34]]
; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP41]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP42]])
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a8756db168b10..4cecec4d4663d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -62,15 +62,17 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP25]])
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP26]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP27]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP22]], [[TMP25]]
+; CHECK-NEXT: [[TMP23:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP27]]
; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
index e0b276eae16e6..464250712b187 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
@@ -50,10 +50,10 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
; CHECK-NEXT: [[TMP59:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; CHECK-NEXT: [[TMP37:%.*]] = freeze <vscale x 16 x i1> [[TMP32]]
; CHECK-NEXT: [[TMP38:%.*]] = freeze <vscale x 16 x i1> [[TMP30]]
-; CHECK-NEXT: [[TMP54:%.*]] = or <vscale x 16 x i1> [[TMP37]], [[TMP38]]
; CHECK-NEXT: [[TMP60:%.*]] = freeze <vscale x 16 x i1> [[TMP31]]
-; CHECK-NEXT: [[TMP62:%.*]] = or <vscale x 16 x i1> [[TMP54]], [[TMP60]]
; CHECK-NEXT: [[TMP34:%.*]] = freeze <vscale x 16 x i1> [[TMP59]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <vscale x 16 x i1> [[TMP37]], [[TMP38]]
+; CHECK-NEXT: [[TMP62:%.*]] = or <vscale x 16 x i1> [[TMP24]], [[TMP60]]
; CHECK-NEXT: [[TMP33:%.*]] = or <vscale x 16 x i1> [[TMP62]], [[TMP34]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP33]])
; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 65ac9fda09c33..93c3152cbcc41 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -40,15 +40,17 @@ define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0)
; CHECK-NEXT: [[TMP18:%.*]] = fmul float 0.000000e+00, [[TMP59]]
; CHECK-NEXT: [[TMP19:%.*]] = fmul float 0.000000e+00, [[TMP60]]
; CHECK-NEXT: [[TMP20:%.*]] = fmul float 0.000000e+00, [[TMP61]]
-; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP17]])
-; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP18]])
-; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP19]])
-; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP20]])
+; CHECK-NEXT: [[TMP25:%.*]] = freeze float [[TMP17]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze float [[TMP18]]
+; CHECK-NEXT: [[TMP29:%.*]] = freeze float [[TMP19]]
+; CHECK-NEXT: [[TMP44:%.*]] = freeze float [[TMP20]]
+; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP25]])
+; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP26]])
+; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP29]])
+; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP44]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
-; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP20]]
-; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP25]]
-; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP26]]
+; CHECK-NEXT: [[TMP27:%.*]] = fcmp uno float [[TMP25]], [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = fcmp uno float [[TMP29]], [[TMP44]]
; CHECK-NEXT: [[TMP76:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP78:%.*]] = or i1 [[TMP76]], [[TMP77]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 8e71583ef56af..beee8ddce5f13 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -64,13 +64,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -342,14 +342,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -511,13 +511,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1183,13 +1183,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index 5e31279f2a013..2c304bcef02fb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -33,16 +33,17 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP9:%[0-9]+]]>
-; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP8:%[0-9]+]]>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%ld.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: WIDEN ir<%select.cmp> = icmp slt ir<%a>, ir<%ld>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of ir<%select.cmp>
-; CHECK-NEXT: EMIT vp<[[VP8]]> = select vp<[[VP7]]>, ir<%select.cmp>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP7]]>, ir<%ld>, ir<%data.phi>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze ir<%select.cmp>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP8]]>, ir<%select.cmp>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP8]]>, ir<%ld>, ir<%data.phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -50,18 +51,18 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP10]]>, vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -93,8 +94,8 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP14:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP15:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
@@ -103,9 +104,10 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: WIDEN ir<%ld> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-TF-NEXT: WIDEN ir<%select.cmp> = icmp slt ir<%a>, ir<%ld>
; CHECK-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%select.cmp>
-; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = any-of vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP12]]>, vp<[[VP11]]>, vp<[[VP6]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP12]]>, ir<%ld>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = freeze vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP12]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP13]]>, ir<%ld>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -113,11 +115,11 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP14]]>, vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP16]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP17]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
@@ -179,8 +181,8 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP10:%[0-9]+]]>, if.then.0 ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP12:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP11:%[0-9]+]]>, if.then.0 ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%a.addr> = getelementptr inbounds nuw ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw i32, ir<%a.addr>, ir<1>
@@ -206,9 +208,10 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%if.cond>
-; CHECK-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP9]]>, ir<%if.cond>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP9]]>, vp<[[VP8]]>, ir<%data.phi>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%if.cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP10]]>, ir<%if.cond>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP10]]>, vp<[[VP8]]>, ir<%data.phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -216,18 +219,18 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP11]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP13]]>, middle.block ], [ ir<%default_val>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP14]]>, middle.block ], [ ir<%default_val>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -256,8 +259,8 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP15:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP16:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP15:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
@@ -269,9 +272,10 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: CLONE ir<%b.addr> = getelementptr ir<%b>, vp<[[VP9]]>
; CHECK-TF-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer i32, ir<%b.addr>, ir<1>
; CHECK-TF-NEXT: WIDEN ir<%ld.b> = load vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP6]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP13]]>, ir<%ld.b>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = freeze vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = any-of vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP14]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP16]]> = select vp<[[VP14]]>, ir<%ld.b>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -279,11 +283,11 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP15]]>, vp<[[VP14]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP16]]>, vp<[[VP15]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP17]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP18]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 70bed19a17a62..fdc571c281c02 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -27,34 +27,35 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -116,34 +117,35 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -201,37 +203,38 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part ir<%data>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
index d5e9bb529a487..0ad003fc76991 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
@@ -32,39 +32,41 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-EMPTY:
; IC2-NEXT: vector.body:
; IC2-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP11:%[0-9]+]]>
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP12:%[0-9]+]]>
-; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
-; IC2-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP10:%[0-9]+]]>, vector.body ]
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP13:%[0-9]+]]>
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP14:%[0-9]+]]>
+; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP11:%[0-9]+]]>, vector.body ]
+; IC2-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP12:%[0-9]+]]>, vector.body ]
; IC2-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
; IC2-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%ld.addr>, ir<1>, ir<4>
; IC2-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>
; IC2-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP7]]>
; IC2-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP4]]>, ir<%ld>
; IC2-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP4]]>, ir<%ld>.1
-; IC2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of ir<%select.cmp>, ir<%select.cmp>.1
-; IC2-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP8]]>, ir<%select.cmp>, vp<[[VP5]]>
-; IC2-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP8]]>, ir<%select.cmp>.1, vp<[[VP6]]>
-; IC2-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP8]]>, ir<%ld>, ir<%data.phi>
-; IC2-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP8]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%select.cmp>
+; IC2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%select.cmp>.1
+; IC2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP8]]>, vp<[[VP9]]>
+; IC2-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP10]]>, ir<%select.cmp>, vp<[[VP5]]>
+; IC2-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP10]]>, ir<%select.cmp>.1, vp<[[VP6]]>
+; IC2-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP10]]>, ir<%ld>, ir<%data.phi>
+; IC2-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP10]]>, ir<%ld>.1, ir<%data.phi>.1
; IC2-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<8>
-; IC2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; IC2-NEXT: EMIT branch-on-cond vp<[[VP13]]>
+; IC2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC2-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; IC2-NEXT: Successor(s): middle.block, vector.body
; IC2-EMPTY:
; IC2-NEXT: middle.block:
-; IC2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP11]]>, vp<[[VP9]]>, vp<[[VP12]]>, vp<[[VP10]]>
+; IC2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP14]]>, vp<[[VP12]]>
; IC2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<%n.vec>
; IC2-NEXT: EMIT branch-on-cond vp<%cmp.n>
; IC2-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; IC2-EMPTY:
; IC2-NEXT: ir-bb<exit>:
-; IC2-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP15]]> from middle.block)
+; IC2-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP17]]> from middle.block)
; IC2-NEXT: No successors
; IC2-EMPTY:
; IC2-NEXT: ir-bb<scalar.ph>:
; IC2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; IC2-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP15]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
+; IC2-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP17]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
; IC2-NEXT: Successor(s): ir-bb<loop>
; IC2-EMPTY:
; IC2-NEXT: ir-bb<loop>:
@@ -97,10 +99,10 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-EMPTY:
; IC2-TF-NEXT: vector.body:
; IC2-TF-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP19:%[0-9]+]]>
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP20:%[0-9]+]]>
-; IC2-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP17:%[0-9]+]]>, vector.body ]
-; IC2-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP18:%[0-9]+]]>, vector.body ]
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP21:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP22:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP19:%[0-9]+]]>, vector.body ]
+; IC2-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP20:%[0-9]+]]>, vector.body ]
; IC2-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast vp<%index>
; IC2-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = step-vector i64
; IC2-TF-NEXT: EMIT vp<%vec.iv> = add nuw vp<[[VP7]]>, vp<[[VP8]]>
@@ -117,22 +119,24 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP4]]>, ir<%ld>.1
; IC2-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%select.cmp>
; IC2-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP12]]>, ir<%select.cmp>.1
-; IC2-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = any-of vp<[[VP14]]>, vp<[[VP15]]>
-; IC2-TF-NEXT: EMIT vp<[[VP17]]> = select vp<[[VP16]]>, vp<[[VP14]]>, vp<[[VP5]]>
-; IC2-TF-NEXT: EMIT vp<[[VP18]]> = select vp<[[VP16]]>, vp<[[VP15]]>, vp<[[VP6]]>
-; IC2-TF-NEXT: EMIT vp<[[VP19]]> = select vp<[[VP16]]>, ir<%ld>, ir<%data.phi>
-; IC2-TF-NEXT: EMIT vp<[[VP20]]> = select vp<[[VP16]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC2-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = freeze vp<[[VP14]]>
+; IC2-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = freeze vp<[[VP15]]>
+; IC2-TF-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = any-of vp<[[VP16]]>, vp<[[VP17]]>
+; IC2-TF-NEXT: EMIT vp<[[VP19]]> = select vp<[[VP18]]>, vp<[[VP14]]>, vp<[[VP5]]>
+; IC2-TF-NEXT: EMIT vp<[[VP20]]> = select vp<[[VP18]]>, vp<[[VP15]]>, vp<[[VP6]]>
+; IC2-TF-NEXT: EMIT vp<[[VP21]]> = select vp<[[VP18]]>, ir<%ld>, ir<%data.phi>
+; IC2-TF-NEXT: EMIT vp<[[VP22]]> = select vp<[[VP18]]>, ir<%ld>.1, ir<%data.phi>.1
; IC2-TF-NEXT: EMIT vp<%index.next> = add vp<%index>, ir<8>
-; IC2-TF-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; IC2-TF-NEXT: EMIT branch-on-cond vp<[[VP21]]>
+; IC2-TF-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC2-TF-NEXT: EMIT branch-on-cond vp<[[VP23]]>
; IC2-TF-NEXT: Successor(s): middle.block, vector.body
; IC2-TF-EMPTY:
; IC2-TF-NEXT: middle.block:
-; IC2-TF-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP19]]>, vp<[[VP17]]>, vp<[[VP20]]>, vp<[[VP18]]>
+; IC2-TF-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP21]]>, vp<[[VP19]]>, vp<[[VP22]]>, vp<[[VP20]]>
; IC2-TF-NEXT: Successor(s): ir-bb<exit>
; IC2-TF-EMPTY:
; IC2-TF-NEXT: ir-bb<exit>:
-; IC2-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP23]]> from middle.block)
+; IC2-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP25]]> from middle.block)
; IC2-TF-NEXT: No successors
; IC2-TF-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
index 87c0606029738..a13028b1386b3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
@@ -28,10 +28,11 @@ define i64 @multi_exiting_to_
diff erent_exits_live_in_exit_values() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -115,10 +116,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -195,10 +197,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -285,10 +288,11 @@ define i64 @two_early_exits_same_exit_with_constant_live_outs() {
; CHECK-NEXT: WIDEN ir<%ld.B> = load vp<[[VP6]]>
; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%ld.A>, ir<%ld.B>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-or ir<%cmp1>, ir<%cmp2>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit.check, middle.block
@@ -308,12 +312,12 @@ define i64 @two_early_exits_same_exit_with_constant_live_outs() {
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: vector.early.exit.0:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ] (extra operands: ir<43> from middle.block, vp<[[VP14]]> from vector.early.exit.0, ir<100> from vector.early.exit.1)
+; CHECK-NEXT: IR %retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ] (extra operands: ir<43> from middle.block, vp<[[VP15]]> from vector.early.exit.0, ir<100> from vector.early.exit.1)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
index eab48211bf8d8..277b75c867ca1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
@@ -25,15 +25,17 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI]], <16 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI1]], <16 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP6]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI2]], <16 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI3]], <16 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <16 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <16 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <16 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <16 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP4]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI]], <16 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP5]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI1]], <16 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP6]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI2]], <16 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP7]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI3]], <16 x float> [[TMP23]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <16 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP11:%.*]] = freeze <16 x i1> [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <16 x float> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fcmp uno <16 x float> [[TMP22]], [[TMP23]]
; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP12]])
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 7e9a73d197a10..bdd795ac02cf4 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -59,13 +59,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -331,14 +331,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP27]], ptr align 2 [[TMP29]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -449,13 +449,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1025,13 +1025,13 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1187,13 +1187,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1330,13 +1330,13 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i16> splat (i16 500), [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
index 63aff9d62f21e..1ceb488c17683 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
@@ -18,10 +18,10 @@ define float @fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
@@ -83,10 +83,10 @@ define float @fminnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float 9.900000e+02), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
@@ -148,10 +148,10 @@ define i64 @dead_fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
index b8f1c857888ce..3e1a8b68bb2aa 100644
--- a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
@@ -24,14 +24,15 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC3-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
; IC3-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; IC3-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; IC3-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC3-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD3]])
-; IC3-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD4]])
+; IC3-NEXT: [[TMP6:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC3-NEXT: [[TMP7:%.*]] = freeze <4 x float> [[WIDE_LOAD3]]
+; IC3-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; IC3-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP6]])
+; IC3-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]])
+; IC3-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP8]])
; IC3-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 12
-; IC3-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
-; IC3-NEXT: [[TMP7:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; IC3-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP7]]
-; IC3-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP8]]
+; IC3-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP8]], [[TMP8]]
+; IC3-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP6]], [[TMP7]]
; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
; IC3-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
; IC3-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -78,15 +79,17 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; IC4-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC4-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD4]])
-; IC4-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; IC4-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; IC4-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC4-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; IC4-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; IC4-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; IC4-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP9]])
+; IC4-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP10]])
+; IC4-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP11]])
+; IC4-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]])
; IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; IC4-NEXT: [[TMP24:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; IC4-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; IC4-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP24]]
-; IC4-NEXT: [[TMP28:%.*]] = freeze <4 x i1> [[TMP25]]
+; IC4-NEXT: [[TMP26:%.*]] = fcmp uno <4 x float> [[TMP9]], [[TMP10]]
+; IC4-NEXT: [[TMP28:%.*]] = fcmp uno <4 x float> [[TMP11]], [[TMP8]]
; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP26]], [[TMP28]]
; IC4-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP29]])
; IC4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -138,19 +141,21 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC5-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; IC5-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; IC5-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
-; IC5-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC5-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD5]])
-; IC5-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD6]])
-; IC5-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD7]])
-; IC5-NEXT: [[TMP9]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[WIDE_LOAD8]])
+; IC5-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC5-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; IC5-NEXT: [[TMP12:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; IC5-NEXT: [[TMP13:%.*]] = freeze <4 x float> [[WIDE_LOAD7]]
+; IC5-NEXT: [[TMP14:%.*]] = freeze <4 x float> [[WIDE_LOAD8]]
+; IC5-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP10]])
+; IC5-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP11]])
+; IC5-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP12]])
+; IC5-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP13]])
+; IC5-NEXT: [[TMP9]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]])
; IC5-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 20
-; IC5-NEXT: [[TMP14:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD8]], [[WIDE_LOAD8]]
-; IC5-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD5]]
-; IC5-NEXT: [[TMP13:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD7]]
-; IC5-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP12]]
-; IC5-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP13]]
+; IC5-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[TMP14]], [[TMP14]]
+; IC5-NEXT: [[TMP18:%.*]] = fcmp uno <4 x float> [[TMP10]], [[TMP11]]
+; IC5-NEXT: [[TMP20:%.*]] = fcmp uno <4 x float> [[TMP12]], [[TMP13]]
; IC5-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP18]], [[TMP20]]
-; IC5-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP14]]
; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP21]], [[TMP22]]
; IC5-NEXT: [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP23]])
; IC5-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8f2e0297aab7d..9557c1167abfb 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -56,11 +56,12 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
@@ -133,15 +134,17 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i64 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp uno <4 x float> [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP17:%.*]] = fcmp uno <4 x float> [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -268,7 +271,7 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x float> [[TMP20]], float [[TMP24]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
; CHECK: [[PRED_LOAD_CONTINUE7]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x float> [ [[TMP20]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[TMP56:%.*]] = phi <4 x float> [ [[TMP20]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], %[[PRED_LOAD_IF6]] ]
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
; CHECK: [[PRED_LOAD_IF8]]:
@@ -308,13 +311,14 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP49:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP48]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
; CHECK: [[PRED_LOAD_CONTINUE15]]:
-; CHECK-NEXT: [[TMP50:%.*]] = phi <4 x float> [ [[TMP44]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP49]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP65:%.*]] = phi <4 x float> [ [[TMP44]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP49]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[TMP56]]
+; CHECK-NEXT: [[TMP50:%.*]] = freeze <4 x float> [[TMP65]]
; CHECK-NEXT: [[TMP51]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP26]])
; CHECK-NEXT: [[TMP52]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP50]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP55:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP50]]
-; CHECK-NEXT: [[TMP56:%.*]] = freeze <4 x i1> [[TMP55]]
-; CHECK-NEXT: [[TMP57:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP56]])
+; CHECK-NEXT: [[TMP57:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
; CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP59:%.*]] = or i1 [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index b0dd95735c5f3..6cca79fd5ea94 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -203,10 +203,10 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP2]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
@@ -268,10 +268,10 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
@@ -335,10 +335,10 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
; CHECK-NEXT: [[IV:%.*]] = add nuw i64 10, [[INDEX]]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP5]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
@@ -403,10 +403,10 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
; CHECK-NEXT: [[IV:%.*]] = add nuw i64 [[START]], [[INDEX]]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP5]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
@@ -694,11 +694,12 @@ define float @test_fmax_and_fmax(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC_0]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
-; CHECK-NEXT: [[TMP2]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP11]]
@@ -813,12 +814,12 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 {
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP21]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x float> [ [[TMP17]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x float> [ [[TMP17]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x float> [[TMP26]]
; CHECK-NEXT: [[TMP24]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP23]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]]
-; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP26]])
+; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 24999536d936c..a6a02fdc736c1 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -203,10 +203,10 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP2]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
@@ -268,10 +268,10 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index d1ef3a607a243..8637897644cae 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -15,13 +15,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
index 9a6e990b7a84c..967c70f094346 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -373,10 +373,10 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD8]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP33]]
-; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = or <4 x i1> [[TMP5]], [[TMP16]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP37]]
-; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP17]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP5]], [[TMP16]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP38]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP19]], [[TMP20]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]])
; CHECK-VF4IC4-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -470,10 +470,10 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp sgt i32 [[TMP22]], 3
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[CMP1]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
-; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP49]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[CMP1]], i1 [[TMP3]]
; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP4]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index 58ffa4c050973..ede9f3eb7ce04 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -777,10 +777,10 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP33]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP34]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -868,10 +868,10 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
-; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
-; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP46:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP46]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
@@ -1269,10 +1269,10 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP29]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1335,10 +1335,10 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
-; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
-; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP43]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
@@ -1486,10 +1486,10 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP39:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP39]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1583,10 +1583,10 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
-; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
-; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP55:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP55]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
@@ -1979,10 +1979,10 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP39:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP39]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -2082,10 +2082,10 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP3]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
-; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
-; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP55:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP55]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP3]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
@@ -2267,10 +2267,10 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[STEP_ADD_3]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP18]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP19]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; CHECK-VF4IC4-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> [[TMP2]]
@@ -2348,10 +2348,10 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP8]], 0
; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = freeze i1 [[TMP14]]
; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = freeze i1 [[TMP16]]
-; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]]
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = freeze i1 [[TMP17]]
+; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP18]], [[TMP19]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = or i1 [[TMP38]], [[TMP21]]
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = or i1 [[TMP22]], [[TMP23]]
; CHECK-VF1IC4-NEXT: [[TMP25]] = select i1 [[TMP24]], i1 [[TMP14]], i1 [[TMP2]]
; CHECK-VF1IC4-NEXT: [[TMP26]] = select i1 [[TMP24]], i1 [[TMP15]], i1 [[TMP3]]
@@ -2507,10 +2507,10 @@ define i32 @select_icmp_truncated_iv_also_widened(ptr noalias %a, ptr noalias %b
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = freeze <4 x i1> [[TMP6]]
-; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP29]], [[TMP30]]
; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = freeze <4 x i1> [[TMP7]]
-; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP8]]
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP29]], [[TMP30]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP33]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]])
; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP20]], <4 x i1> [[TMP5]], <4 x i1> [[TMP9]]
@@ -2603,10 +2603,10 @@ define i32 @select_icmp_truncated_iv_also_widened(ptr noalias %a, ptr noalias %b
; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = icmp sgt i32 [[TMP12]], 3
; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = freeze i1 [[TMP13]]
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = freeze i1 [[TMP14]]
-; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = or i1 [[TMP44]], [[TMP26]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = or i1 [[TMP44]], [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP49]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP31]], i1 [[TMP13]], i1 [[TMP1]]
; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP31]], i1 [[TMP14]], i1 [[TMP17]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
index 843b58d1d6012..efbbe480e0041 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
@@ -1969,10 +1969,10 @@ define float @not_vectorized_select_float_induction_icmp(ptr %a, ptr %b, float %
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = icmp sgt <4 x i64> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP15]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP16]]
-; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP41:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP41]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP22]], [[TMP23]]
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-VF4IC4-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index bc6ce82606cf2..eb490e74d9f41 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -771,8 +771,11 @@ for.body:
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK: <2 x float> @llvm.minnum.v2f32
-; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
+; CHECK: [[FREEZE0:%.*]] = freeze <2 x float> [[OP0:%.+]]
+; CHECK-NEXT: [[FREEZE1:%.*]] = freeze <2 x float> [[OP1:%.+]]
+; CHECK-NEXT: <2 x float> @llvm.minnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE0]])
+; CHECK-NEXT: <2 x float> @llvm.minnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE1]])
+; CHECK: fcmp uno <2 x float> [[FREEZE0]], [[FREEZE1]]
define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
@@ -793,8 +796,11 @@ for.body:
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK: <2 x float> @llvm.maxnum.v2f32
-; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
+; CHECK: [[FREEZE0:%.*]] = freeze <2 x float> [[OP0:%.+]]
+; CHECK-NEXT: [[FREEZE1:%.*]] = freeze <2 x float> [[OP1:%.+]]
+; CHECK-NEXT: <2 x float> @llvm.maxnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE0]])
+; CHECK-NEXT: <2 x float> @llvm.maxnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE1]])
+; CHECK: fcmp uno <2 x float> [[FREEZE0]], [[FREEZE1]]
define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index c572f15086082..4523ba2d7d3e7 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -16,7 +16,8 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
@@ -79,7 +80,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: store i16 [[TMP30]], ptr [[TMP34]], align 2
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index 7969a080678ef..00bf9310e7607 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -1167,10 +1167,10 @@ define float @select_const_f32_from_icmp(ptr %v, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP3]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP33]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
@@ -1245,10 +1245,10 @@ define float @select_const_f32_from_icmp(ptr %v, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = icmp ne i32 [[TMP14]], 3
; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP15]]
; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = freeze i1 [[TMP16]]
-; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = or i1 [[TMP19]], [[TMP20]]
; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = freeze i1 [[TMP17]]
-; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = or i1 [[TMP21]], [[TMP22]]
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[TMP18]]
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = or i1 [[TMP19]], [[TMP20]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = or i1 [[TMP39]], [[TMP22]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = or i1 [[TMP23]], [[TMP24]]
; CHECK-VF1IC4-NEXT: [[TMP26]] = select i1 [[TMP25]], i1 [[TMP15]], i1 [[TMP0]]
; CHECK-VF1IC4-NEXT: [[TMP27]] = select i1 [[TMP25]], i1 [[TMP16]], i1 [[TMP1]]
@@ -1395,10 +1395,10 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP17]], [[TMP24]]
; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP33]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = or <4 x i1> [[TMP17]], [[TMP24]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1483,10 +1483,10 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp ne i32 [[TMP14]], 3
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP23]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
-; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP47]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[TMP23]], i1 [[TMP0]]
; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
new file mode 100644
index 0000000000000..53971f07d617c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+; Previously this applied the
+; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
+; combine and removed the freezes, which was incorrect.
+;
+
+define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
+; CHECK-LABEL: define i1 @find_nan(
+; CHECK-SAME: ptr align 16 dereferenceable(32) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM:.*]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i1 [ true, %[[VECTOR_EARLY_EXIT]] ], [ false, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i1 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds float, ptr %a, i64 %iv
+ %x = load float, ptr %gep
+ %isnan = fcmp uno float %x, %x
+ br i1 %isnan, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i1 [ true, %loop ], [ false, %latch ]
+ ret i1 %res
+}
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
index 082c2dcb5d6d4..2ffafcb118f24 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
@@ -27,10 +27,10 @@ define i64 @multi_exiting_to_
diff erent_exits_live_in_exit_values() {
; VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP2]]
; VF4IC4-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP6]]
-; VF4IC4-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
; VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP7]]
-; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP10]], [[TMP15]]
; VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP8]]
+; VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
+; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP18]], [[TMP15]]
; VF4IC4-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
index 749c4d93613b6..776d02f0c70a7 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
@@ -27,10 +27,10 @@ define i64 @multi_exiting_to_
diff erent_exits_live_in_exit_values() {
; VF4IC4-NEXT: [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
; VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP8]]
; VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP6]]
-; VF4IC4-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP17]], [[TMP13]]
; VF4IC4-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP7]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP9]], [[TMP10]]
; VF4IC4-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP14]]
+; VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP17]], [[TMP13]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP18]], [[TMP10]]
; VF4IC4-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP15]], [[TMP16]]
; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -108,10 +108,10 @@ define i64 @same_exit_block_pre_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -194,10 +194,10 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
; VF4IC4-NEXT: [[TMP29:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], splat (i8 72)
; VF4IC4-NEXT: [[TMP35:%.*]] = freeze <4 x i1> [[TMP17]]
; VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP14]]
-; VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP35]], [[TMP13]]
; VF4IC4-NEXT: [[TMP32:%.*]] = freeze <4 x i1> [[TMP28]]
-; VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; VF4IC4-NEXT: [[TMP34:%.*]] = freeze <4 x i1> [[TMP29]]
+; VF4IC4-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP35]], [[TMP13]]
+; VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP12]], [[TMP32]]
; VF4IC4-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP33]], [[TMP34]]
; VF4IC4-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP9]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -286,10 +286,10 @@ define i64 @same_exit_block_post_inc_use() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -382,10 +382,10 @@ define i64 @
diff _exit_block_pre_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -483,10 +483,10 @@ define i64 @
diff _exit_block_post_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -594,10 +594,10 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
; VF4IC4-NEXT: [[TMP43:%.*]] = icmp ne <4 x i8> [[REVERSE6]], [[REVERSE16]]
; VF4IC4-NEXT: [[TMP48:%.*]] = freeze <4 x i1> [[TMP21]]
; VF4IC4-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP19]]
-; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP48]], [[TMP22]]
; VF4IC4-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[TMP20]]
-; VF4IC4-NEXT: [[TMP45:%.*]] = or <4 x i1> [[TMP23]], [[TMP44]]
; VF4IC4-NEXT: [[TMP47:%.*]] = freeze <4 x i1> [[TMP43]]
+; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP48]], [[TMP22]]
+; VF4IC4-NEXT: [[TMP45:%.*]] = or <4 x i1> [[TMP23]], [[TMP44]]
; VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP45]], [[TMP47]]
; VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -704,10 +704,10 @@ define i8 @same_exit_block_use_loaded_value() {
; VF4IC4-NEXT: [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
; VF4IC4-NEXT: [[TMP43:%.*]] = freeze <4 x i1> [[TMP12]]
; VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP29]]
-; VF4IC4-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP43]], [[TMP18]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP13]], [[TMP14]]
; VF4IC4-NEXT: [[TMP30:%.*]] = freeze <4 x i1> [[TMP17]]
+; VF4IC4-NEXT: [[TMP44:%.*]] = or <4 x i1> [[TMP43]], [[TMP18]]
+; VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP44]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP19]], [[TMP30]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -822,10 +822,10 @@ define i8 @same_exit_block_reverse_use_loaded_value() {
; VF4IC4-NEXT: [[TMP37:%.*]] = icmp ne <4 x i8> [[REVERSE7]], [[REVERSE15]]
; VF4IC4-NEXT: [[TMP54:%.*]] = freeze <4 x i1> [[TMP21]]
; VF4IC4-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP19]]
-; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP54]], [[TMP22]]
; VF4IC4-NEXT: [[TMP51:%.*]] = freeze <4 x i1> [[TMP20]]
-; VF4IC4-NEXT: [[TMP52:%.*]] = or <4 x i1> [[TMP23]], [[TMP51]]
; VF4IC4-NEXT: [[TMP53:%.*]] = freeze <4 x i1> [[TMP37]]
+; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP54]], [[TMP22]]
+; VF4IC4-NEXT: [[TMP52:%.*]] = or <4 x i1> [[TMP23]], [[TMP51]]
; VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP52]], [[TMP53]]
; VF4IC4-NEXT: [[TMP25:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index 58ff87764459a..e4633bcb5568a 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -18,13 +18,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
More information about the llvm-commits
mailing list