[llvm] [VPlan] Fix VPInstruction::AnyOf combine undoing freeze (PR #223640)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 01:35:26 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/223640
>From be907e26533bc3436dfc10d42cf7997350a6c8b3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 16:20:23 +0800
Subject: [PATCH 1/9] Precommit test
---
.../single-early-exit-anyof-fold.ll | 54 +++++++++++++++++++
1 file changed, 54 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
new file mode 100644
index 00000000000000..ca061fc0ef6e20
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+; FIXME; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
+; replaces two individual freezes with one joint freeze which is incorrect.
+;
+; e.g. when x[0] = NaN and x[4] = poison the scalar loop is well defined and
+; returns true. When vectorized we get reduce.or(freeze(fcmp uno x[0]=NaN, x[4]=poison))
+; which may return false.
+
+define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
+; CHECK-LABEL: define i1 @find_nan(
+; CHECK-SAME: ptr align 16 dereferenceable(32) [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM:.*]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i1 [ true, %[[VECTOR_EARLY_EXIT]] ], [ false, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i1 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds float, ptr %a, i64 %iv
+ %x = load float, ptr %gep
+ %isnan = fcmp uno float %x, %x
+ br i1 %isnan, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i1 [ true, %loop ], [ false, %latch ]
+ ret i1 %res
+}
>From d2e9e5d45afad2c4cdcb5bb409db47d7706b3125 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 17:18:12 +0800
Subject: [PATCH 2/9] [VPlan] Fix VPInstruction::AnyOf combine undoing freeze
There is an any-of combine for unrolled VPlans which does:
any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
However any-of implicitly freezes each individual operand and this means we go from `freeze (fcmp uno A, A)` to `freeze (fcmp uno A, B)` which isn't sound: alive2.llvm.org/ce/z/UdQM7C
This causes miscompiles today with early exit loops, see the attached test case in single-early-exit-anyof-fold.ll.
This fixes it by explicitly modelling the freeze in VPlan. There are three places where we use AnyOf:
1) early exit loops: the freeze needs to be applied per-lane, so apply it to `(any-of (freeze (combined-conds-to-exit)))`
2) handleMaxMinNumReductions: If any lane of the reduction was poison in the scalar loop, the final result will be poison. We only need to freeze the result of AnyOf to prevent immediate UB when branching. Freezing individual operands blocks the any-of combine otherwise.
3) handleFindLastReductions: we need to freeze Cond itself since it's got multiple uses, but I plan on fixing this in a separate PR
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +-
.../Vectorize/VPlanConstruction.cpp | 8 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 +++-
.../LoopVectorize/AArch64/cmp_cost.ll | 99 ++++++++++---------
.../AArch64/fmax-without-fast-math-flags.ll | 60 ++++++-----
.../AArch64/fmin-without-fast-math-flags.ll | 23 +++--
.../AArch64/single-early-exit-interleave.ll | 4 +-
.../sve-non-native-interleaved-accesses.ll | 5 +-
.../conditional-scalar-assignment-vplan.ll | 66 +++++++------
.../VPlan/early_exit_with_stores_vplan.ll | 57 ++++++-----
...ave-conditional-scalar-assignment-vplan.ll | 58 ++++++-----
.../VPlan/uncountable-early-exit-vplan.ll | 34 ++++---
.../LoopVectorize/X86/cost-any-of.ll | 21 ++--
...epilog-vectorization-fmaxnum-reductions.ll | 12 +--
.../LoopVectorize/fcmp-uno-fold-interleave.ll | 68 ++++++-------
...fmax-without-fast-math-flags-interleave.ll | 31 +++---
.../fmax-without-fast-math-flags.ll | 24 ++---
.../fmin-without-fast-math-flags.ll | 8 +-
.../iv-select-cmp-non-const-iv-start.ll | 8 +-
.../LoopVectorize/iv-select-cmp-trunc.ll | 48 ++++-----
.../Transforms/LoopVectorize/iv-select-cmp.ll | 4 +-
.../Transforms/LoopVectorize/select-cmp.ll | 16 +--
.../single-early-exit-anyof-fold.ll | 7 +-
.../single-early-exit-interleave-hint.ll | 4 +-
.../single-early-exit-interleave.ll | 36 +++----
26 files changed, 380 insertions(+), 354 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index daabca902724b1..d570002fb82367 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1378,8 +1378,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Returns a scalar boolean value, which is true if any lane of its
// (boolean) vector operands is true. It produces the reduced value across
// all unrolled iterations. Unrolling will add all copies of its original
- // operand as additional operands. AnyOf is poison-safe as all operands
- // will be frozen.
+ // operand as additional operands.
AnyOf,
// Calculates the first active lane index of the vector predicate operands.
// It produces the lane index across all unrolled iterations. Unrolling will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4cd455adb7a902..4b1ebed273dcff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1695,7 +1695,9 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
}
VPValue *AnyNaNLane =
- LatchBuilder.createNaryOp(VPInstruction::AnyOf, {AllNaNLanes});
+ LatchBuilder.createNaryOp(VPInstruction::AnyOf, AllNaNLanes);
+ // Freeze to prevent immediate UB from branching on poison.
+ AnyNaNLane = LatchBuilder.createFreeze(AnyNaNLane);
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->begin());
for (const auto &[RedPhiR, _] : MinOrMaxNumReductionsToHandle) {
@@ -1874,7 +1876,9 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
if (HeaderMask)
Cond = Builder.createLogicalAnd(HeaderMask, Cond);
- VPValue *AnyOf = Builder.createNaryOp(VPInstruction::AnyOf, {Cond});
+ VPValue *AnyOf =
+ Builder.createNaryOp(VPInstruction::AnyOf, Builder.createFreeze(Cond));
+ // FIXME: The Cond here needs to be frozen too.
VPValue *MaskSelect = Builder.createSelect(AnyOf, Cond, MaskPHI);
MaskPHI->addIncoming(MaskSelect);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5997b6e0fef76d..4f3699e7cb8028 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1036,9 +1036,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::AnyOf: {
- Value *Res = Builder.CreateFreeze(State.get(getOperand(0)));
+ Value *Res = State.get(getOperand(0));
for (VPValue *Op : drop_begin(operands()))
- Res = Builder.CreateOr(Res, Builder.CreateFreeze(State.get(Op)));
+ Res = Builder.CreateOr(Res, State.get(Op));
return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
}
case VPInstruction::ExtractLane: {
@@ -1741,9 +1741,15 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return Op == getOperand(1) || Op == getOperand(2);
case Instruction::PHI:
return true;
+ case Instruction::Select:
+ if (Op == getOperand(0) && Op != getOperand(1) && Op != getOperand(2) &&
+ vputils::isSingleScalar(Op))
+ // TODO: Relax for other opcodes.
+ if (match(Op, m_Freeze(m_VPValue())))
+ return true;
+ [[fallthrough]];
case Instruction::FCmp:
case Instruction::ICmp:
- case Instruction::Select:
case Instruction::Or:
case Instruction::Freeze:
case VPInstruction::Not:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d78a3d3b1e3443..65406fd8f74e85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3086,9 +3086,10 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// Find the uncountable loop exit condition.
VPValue *UncountableCondition = nullptr;
- if (!match(LatchVPBB->getTerminator(),
- m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
- m_VPValue())))
+ if (!match(
+ LatchVPBB->getTerminator(),
+ m_BranchOnTwoConds(m_AnyOf(m_Freeze(m_VPValue(UncountableCondition))),
+ m_VPValue())))
return nullptr;
SmallVector<VPValue *, 4> Worklist;
@@ -3407,8 +3408,15 @@ bool VPlanTransforms::handleUncountableEarlyExits(
for (const EarlyExitInfo &Info : drop_begin(Exits))
Combined = LatchBuilder.createLogicalOr(Combined, Info.CondToExit);
- VPValue *IsAnyExitTaken =
- LatchBuilder.createNaryOp(VPInstruction::AnyOf, {Combined});
+ // Even though the logical or prevents per-lane posion propagation, we need to
+ // freeze Combined to prevent poisoning the entire AnyOf result:
+ //
+ // Exits[0].CondToExit = [0,1,0,0]
+ // Exits[1].CondToExit = [0,0,p,p]
+ // Combined = [0,1,p,p]
+ // AnyOf = 1
+ VPValue *IsAnyExitTaken = LatchBuilder.createNaryOp(
+ VPInstruction::AnyOf, LatchBuilder.createFreeze(Combined));
// Create a comparison for the latch exit condition and replace the
// BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index a25edccb605145..fee76962809427 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -19,13 +19,14 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i32 %iv to float
@@ -33,14 +34,14 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i32 %iv.next, %n
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
@@ -49,13 +50,14 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP4]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP5]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP6]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP7]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP6]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP8]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %iv.f = sitofp i32 %iv to float
@@ -63,14 +65,14 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 4: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 4: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 4: IR %ec = icmp eq i32 %iv.next, %n
-; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP9]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP13]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP14]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: LV: Selecting VF: 4.
;
entry:
@@ -100,13 +102,14 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi double [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i64 %iv to double
@@ -114,14 +117,14 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 0 for VF 2: IR %max.next = call double @llvm.maxnum.f64(double %max, double %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i64 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i64 %iv.next, %n
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%max>, ir<%max.next>
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11]]> = select vp<[[VP6]]>, vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP12]]> = compute-reduction-result (fmaxnum) vp<[[VP10]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
-; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = not vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP13]]>
+; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK: LV: Selecting VF: 1.
;
entry:
@@ -182,8 +185,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP34:%[0-9]+]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP35:%[0-9]+]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP36:%[0-9]+]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
; CHECK: Cost of 7 for VF 2: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP37:%[0-9]+]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -240,8 +243,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
; CHECK: Cost of 13 for VF 4: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -298,8 +301,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
; CHECK: Cost of 10 for VF 8: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 8: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -356,8 +359,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
; CHECK: Cost of 20 for VF 16: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 16: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -439,7 +442,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
; CHECK: Cost of 1 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -462,7 +465,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
; CHECK: Cost of 1 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -515,7 +518,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 2: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 8796093022208 (9.537e-05%, estimated))
; CHECK: Cost of 2 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -538,7 +541,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 4: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 8796093022208 (9.537e-05%, estimated))
; CHECK: Cost of 2 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 09ccca93fa3688..715f1a62252b38 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -69,29 +69,28 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
+; CHECK-NEXT: [[TMP14:%.*]] = freeze i1 [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP14]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -174,27 +173,24 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[TMP29:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD13]], [[WIDE_LOAD9]]
; CHECK-NEXT: [[TMP34:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD14]], [[WIDE_LOAD10]]
-; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP35:%.*]] = freeze <4 x i1> [[TMP29]]
-; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP18]], [[TMP35]]
-; CHECK-NEXT: [[TMP41:%.*]] = freeze <4 x i1> [[TMP34]]
-; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP41]]
+; CHECK-NEXT: [[TMP35:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP35]], [[TMP29]]
+; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP34]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP42]])
+; CHECK-NEXT: [[TMP41:%.*]] = freeze i1 [[TMP19]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
-; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP27:%.*]] = or i1 [[TMP41]], [[TMP21]]
+; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
-; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
-; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
+; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP41]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP32]])
; CHECK-NEXT: [[RDX_MINMAX16:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX15]], <4 x float> [[TMP33]])
@@ -204,11 +200,11 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[RDX_MINMAX19:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX18]], <4 x float> [[TMP37]])
; CHECK-NEXT: [[TMP40:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX19]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP41]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi float [ [[TMP40]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a8756db168b107..1221eac37d4d6a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -69,29 +69,28 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
+; CHECK-NEXT: [[TMP14:%.*]] = freeze i1 [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP14]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
index e0b276eae16e6a..464250712b1873 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
@@ -50,10 +50,10 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
; CHECK-NEXT: [[TMP59:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; CHECK-NEXT: [[TMP37:%.*]] = freeze <vscale x 16 x i1> [[TMP32]]
; CHECK-NEXT: [[TMP38:%.*]] = freeze <vscale x 16 x i1> [[TMP30]]
-; CHECK-NEXT: [[TMP54:%.*]] = or <vscale x 16 x i1> [[TMP37]], [[TMP38]]
; CHECK-NEXT: [[TMP60:%.*]] = freeze <vscale x 16 x i1> [[TMP31]]
-; CHECK-NEXT: [[TMP62:%.*]] = or <vscale x 16 x i1> [[TMP54]], [[TMP60]]
; CHECK-NEXT: [[TMP34:%.*]] = freeze <vscale x 16 x i1> [[TMP59]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <vscale x 16 x i1> [[TMP37]], [[TMP38]]
+; CHECK-NEXT: [[TMP62:%.*]] = or <vscale x 16 x i1> [[TMP24]], [[TMP60]]
; CHECK-NEXT: [[TMP33:%.*]] = or <vscale x 16 x i1> [[TMP62]], [[TMP34]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP33]])
; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 65ac9fda09c335..624e904fbfbc08 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -47,9 +47,8 @@ define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP20]]
-; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP25]]
-; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP26]]
-; CHECK-NEXT: [[TMP76:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-NEXT: [[TMP27:%.*]] = or i1 [[TMP25]], [[TMP26]]
+; CHECK-NEXT: [[TMP76:%.*]] = freeze i1 [[TMP27]]
; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP78:%.*]] = or i1 [[TMP76]], [[TMP77]]
; CHECK-NEXT: br i1 [[TMP78]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index 5e31279f2a0139..5a9903b4e6e4e8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -33,16 +33,17 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP9:%[0-9]+]]>
-; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP8:%[0-9]+]]>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%ld.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: WIDEN ir<%select.cmp> = icmp slt ir<%a>, ir<%ld>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of ir<%select.cmp>
-; CHECK-NEXT: EMIT vp<[[VP8]]> = select vp<[[VP7]]>, ir<%select.cmp>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP7]]>, ir<%ld>, ir<%data.phi>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze ir<%select.cmp>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP8]]>, ir<%select.cmp>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP8]]>, ir<%ld>, ir<%data.phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -50,18 +51,18 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP10]]>, vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP12]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -93,8 +94,8 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP14:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP15:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
@@ -103,9 +104,10 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: WIDEN ir<%ld> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-TF-NEXT: WIDEN ir<%select.cmp> = icmp slt ir<%a>, ir<%ld>
; CHECK-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%select.cmp>
-; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = any-of vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP12]]>, vp<[[VP11]]>, vp<[[VP6]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP12]]>, ir<%ld>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = freeze vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP12]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP13]]>, ir<%ld>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -113,11 +115,11 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP14]]>, vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP16]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP17]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
@@ -179,8 +181,8 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP10:%[0-9]+]]>, if.then.0 ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP12:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP11:%[0-9]+]]>, if.then.0 ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%a.addr> = getelementptr inbounds nuw ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw i32, ir<%a.addr>, ir<1>
@@ -190,7 +192,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%if.cond>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%if.cond> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
@@ -206,9 +208,10 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%if.cond>
-; CHECK-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP9]]>, ir<%if.cond>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP9]]>, vp<[[VP8]]>, ir<%data.phi>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%if.cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP10]]>, ir<%if.cond>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP10]]>, vp<[[VP8]]>, ir<%data.phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -216,18 +219,18 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP11]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP13]]>, middle.block ], [ ir<%default_val>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP14]]>, middle.block ], [ ir<%default_val>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -256,8 +259,8 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP15:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP16:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP15:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
@@ -269,9 +272,10 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: CLONE ir<%b.addr> = getelementptr ir<%b>, vp<[[VP9]]>
; CHECK-TF-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer i32, ir<%b.addr>, ir<1>
; CHECK-TF-NEXT: WIDEN ir<%ld.b> = load vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP11]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP6]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP13]]>, ir<%ld.b>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = freeze vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = any-of vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP14]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP16]]> = select vp<[[VP14]]>, ir<%ld.b>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -279,11 +283,11 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP15]]>, vp<[[VP14]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP16]]>, vp<[[VP15]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP17]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP18]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 70bed19a17a629..2fb9d906e22cb2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -35,26 +35,27 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -124,26 +125,27 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -209,29 +211,30 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part ir<%data>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
index d5e9bb529a487b..0ad003fc76991e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
@@ -32,39 +32,41 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-EMPTY:
; IC2-NEXT: vector.body:
; IC2-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP11:%[0-9]+]]>
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP12:%[0-9]+]]>
-; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
-; IC2-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP10:%[0-9]+]]>, vector.body ]
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP13:%[0-9]+]]>
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP14:%[0-9]+]]>
+; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP11:%[0-9]+]]>, vector.body ]
+; IC2-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP12:%[0-9]+]]>, vector.body ]
; IC2-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
; IC2-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%ld.addr>, ir<1>, ir<4>
; IC2-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>
; IC2-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP7]]>
; IC2-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP4]]>, ir<%ld>
; IC2-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP4]]>, ir<%ld>.1
-; IC2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of ir<%select.cmp>, ir<%select.cmp>.1
-; IC2-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP8]]>, ir<%select.cmp>, vp<[[VP5]]>
-; IC2-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP8]]>, ir<%select.cmp>.1, vp<[[VP6]]>
-; IC2-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP8]]>, ir<%ld>, ir<%data.phi>
-; IC2-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP8]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%select.cmp>
+; IC2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%select.cmp>.1
+; IC2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP8]]>, vp<[[VP9]]>
+; IC2-NEXT: EMIT vp<[[VP11]]> = select vp<[[VP10]]>, ir<%select.cmp>, vp<[[VP5]]>
+; IC2-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP10]]>, ir<%select.cmp>.1, vp<[[VP6]]>
+; IC2-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP10]]>, ir<%ld>, ir<%data.phi>
+; IC2-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP10]]>, ir<%ld>.1, ir<%data.phi>.1
; IC2-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<8>
-; IC2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; IC2-NEXT: EMIT branch-on-cond vp<[[VP13]]>
+; IC2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC2-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; IC2-NEXT: Successor(s): middle.block, vector.body
; IC2-EMPTY:
; IC2-NEXT: middle.block:
-; IC2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP11]]>, vp<[[VP9]]>, vp<[[VP12]]>, vp<[[VP10]]>
+; IC2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP14]]>, vp<[[VP12]]>
; IC2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<%n.vec>
; IC2-NEXT: EMIT branch-on-cond vp<%cmp.n>
; IC2-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
; IC2-EMPTY:
; IC2-NEXT: ir-bb<exit>:
-; IC2-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP15]]> from middle.block)
+; IC2-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP17]]> from middle.block)
; IC2-NEXT: No successors
; IC2-EMPTY:
; IC2-NEXT: ir-bb<scalar.ph>:
; IC2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; IC2-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP15]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
+; IC2-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP17]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
; IC2-NEXT: Successor(s): ir-bb<loop>
; IC2-EMPTY:
; IC2-NEXT: ir-bb<loop>:
@@ -97,10 +99,10 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-EMPTY:
; IC2-TF-NEXT: vector.body:
; IC2-TF-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP19:%[0-9]+]]>
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP20:%[0-9]+]]>
-; IC2-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP17:%[0-9]+]]>, vector.body ]
-; IC2-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP18:%[0-9]+]]>, vector.body ]
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP21:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP22:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP19:%[0-9]+]]>, vector.body ]
+; IC2-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP20:%[0-9]+]]>, vector.body ]
; IC2-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast vp<%index>
; IC2-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = step-vector i64
; IC2-TF-NEXT: EMIT vp<%vec.iv> = add nuw vp<[[VP7]]>, vp<[[VP8]]>
@@ -117,22 +119,24 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP4]]>, ir<%ld>.1
; IC2-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%select.cmp>
; IC2-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP12]]>, ir<%select.cmp>.1
-; IC2-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = any-of vp<[[VP14]]>, vp<[[VP15]]>
-; IC2-TF-NEXT: EMIT vp<[[VP17]]> = select vp<[[VP16]]>, vp<[[VP14]]>, vp<[[VP5]]>
-; IC2-TF-NEXT: EMIT vp<[[VP18]]> = select vp<[[VP16]]>, vp<[[VP15]]>, vp<[[VP6]]>
-; IC2-TF-NEXT: EMIT vp<[[VP19]]> = select vp<[[VP16]]>, ir<%ld>, ir<%data.phi>
-; IC2-TF-NEXT: EMIT vp<[[VP20]]> = select vp<[[VP16]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC2-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = freeze vp<[[VP14]]>
+; IC2-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = freeze vp<[[VP15]]>
+; IC2-TF-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = any-of vp<[[VP16]]>, vp<[[VP17]]>
+; IC2-TF-NEXT: EMIT vp<[[VP19]]> = select vp<[[VP18]]>, vp<[[VP14]]>, vp<[[VP5]]>
+; IC2-TF-NEXT: EMIT vp<[[VP20]]> = select vp<[[VP18]]>, vp<[[VP15]]>, vp<[[VP6]]>
+; IC2-TF-NEXT: EMIT vp<[[VP21]]> = select vp<[[VP18]]>, ir<%ld>, ir<%data.phi>
+; IC2-TF-NEXT: EMIT vp<[[VP22]]> = select vp<[[VP18]]>, ir<%ld>.1, ir<%data.phi>.1
; IC2-TF-NEXT: EMIT vp<%index.next> = add vp<%index>, ir<8>
-; IC2-TF-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; IC2-TF-NEXT: EMIT branch-on-cond vp<[[VP21]]>
+; IC2-TF-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC2-TF-NEXT: EMIT branch-on-cond vp<[[VP23]]>
; IC2-TF-NEXT: Successor(s): middle.block, vector.body
; IC2-TF-EMPTY:
; IC2-TF-NEXT: middle.block:
-; IC2-TF-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP19]]>, vp<[[VP17]]>, vp<[[VP20]]>, vp<[[VP18]]>
+; IC2-TF-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP21]]>, vp<[[VP19]]>, vp<[[VP22]]>, vp<[[VP20]]>
; IC2-TF-NEXT: Successor(s): ir-bb<exit>
; IC2-TF-EMPTY:
; IC2-TF-NEXT: ir-bb<exit>:
-; IC2-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP23]]> from middle.block)
+; IC2-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP25]]> from middle.block)
; IC2-TF-NEXT: No successors
; IC2-TF-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
index 87c0606029738b..a13028b1386b3a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
@@ -28,10 +28,11 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -115,10 +116,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -195,10 +197,11 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.src>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = any-of ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP7]]>, vp<[[VP8]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
@@ -285,10 +288,11 @@ define i64 @two_early_exits_same_exit_with_constant_live_outs() {
; CHECK-NEXT: WIDEN ir<%ld.B> = load vp<[[VP6]]>
; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%ld.A>, ir<%ld.B>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-or ir<%cmp1>, ir<%cmp2>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): vector.early.exit.check, middle.block
@@ -308,12 +312,12 @@ define i64 @two_early_exits_same_exit_with_constant_live_outs() {
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: vector.early.exit.0:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ] (extra operands: ir<43> from middle.block, vp<[[VP14]]> from vector.early.exit.0, ir<100> from vector.early.exit.1)
+; CHECK-NEXT: IR %retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ] (extra operands: ir<43> from middle.block, vp<[[VP15]]> from vector.early.exit.0, ir<100> from vector.early.exit.1)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
index eab48211bf8d84..88abf190fafbb2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
@@ -32,19 +32,18 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <16 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP11:%.*]] = freeze <16 x i1> [[TMP9]]
-; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP10]], [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP12]])
+; CHECK-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP14]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP15]], [[TMP14]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[INDEX]], i64 128
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 128
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[TMP16]], <16 x float> [[TMP17]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX]], <16 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX7]], <16 x float> [[TMP19]])
@@ -53,7 +52,7 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP22]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RED:%.*]] = phi float [ [[TMP21]], %[[SCALAR_PH]] ], [ [[MIN:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LDV:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
index 63aff9d62f21ea..a5a4cf265a53c3 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
@@ -21,8 +21,8 @@ define float @fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -86,8 +86,8 @@ define float @fminnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -151,8 +151,8 @@ define i64 @dead_fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
index b8f1c857888ce8..19e499443e0f51 100644
--- a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
@@ -30,27 +30,26 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC3-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 12
; IC3-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
; IC3-NEXT: [[TMP7:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; IC3-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP7]]
-; IC3-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP8]]
-; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
+; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
; IC3-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
+; IC3-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP14]]
; IC3-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC3-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]]
-; IC3-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC3-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP15]]
+; IC3-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC3: [[MIDDLE_BLOCK]]:
-; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
-; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
-; IC3-NEXT: [[TMP20:%.*]] = select i1 [[TMP14]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
+; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
+; IC3-NEXT: [[TMP16:%.*]] = select i1 [[TMP10]], i64 [[INDEX]], i64 [[N_VEC]]
; IC3-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP17]], <4 x float> [[TMP18]])
; IC3-NEXT: [[RDX_MINMAX5:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP19]])
; IC3-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX5]])
; IC3-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP14]], true
+; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP10]], true
; IC3-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC3-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC3: [[SCALAR_PH]]:
-; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC3-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC3-NEXT: br label %[[LOOP:.*]]
; IC3: [[LOOP]]:
@@ -85,25 +84,24 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; IC4-NEXT: [[TMP24:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
; IC4-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; IC4-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP24]]
-; IC4-NEXT: [[TMP28:%.*]] = freeze <4 x i1> [[TMP25]]
-; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP26]], [[TMP28]]
+; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP24]], [[TMP25]]
; IC4-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP29]])
+; IC4-NEXT: [[TMP26:%.*]] = freeze i1 [[TMP13]]
; IC4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC4-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP14]]
+; IC4-NEXT: [[TMP15:%.*]] = or i1 [[TMP26]], [[TMP14]]
; IC4-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC4: [[MIDDLE_BLOCK]]:
-; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
-; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
-; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
-; IC4-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
+; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
+; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
+; IC4-NEXT: [[TMP20:%.*]] = select i1 [[TMP26]], i64 [[INDEX]], i64 [[N_VEC]]
; IC4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP16]], <4 x float> [[TMP17]])
; IC4-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP18]])
; IC4-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; IC4-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP13]], true
+; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP26]], true
; IC4-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC4-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC4: [[SCALAR_PH]]:
@@ -147,33 +145,31 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC5-NEXT: [[TMP14:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD8]], [[WIDE_LOAD8]]
; IC5-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD5]]
; IC5-NEXT: [[TMP13:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD7]]
-; IC5-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP12]]
-; IC5-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP13]]
-; IC5-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP18]], [[TMP20]]
-; IC5-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP14]]
-; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP21]], [[TMP22]]
+; IC5-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
+; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
; IC5-NEXT: [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP23]])
+; IC5-NEXT: [[TMP16:%.*]] = freeze i1 [[TMP24]]
; IC5-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC5-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
-; IC5-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC5-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP25]]
+; IC5-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC5: [[MIDDLE_BLOCK]]:
-; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
-; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
-; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
-; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
-; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
-; IC5-NEXT: [[TMP32:%.*]] = select i1 [[TMP24]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
+; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
+; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
+; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
+; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
+; IC5-NEXT: [[TMP26:%.*]] = select i1 [[TMP16]], i64 [[INDEX]], i64 [[N_VEC]]
; IC5-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP27]], <4 x float> [[TMP28]])
; IC5-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP29]])
; IC5-NEXT: [[RDX_MINMAX10:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX9]], <4 x float> [[TMP30]])
; IC5-NEXT: [[RDX_MINMAX11:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX10]], <4 x float> [[TMP31]])
; IC5-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX11]])
; IC5-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP24]], true
+; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP16]], true
; IC5-NEXT: [[TMP35:%.*]] = and i1 [[CMP_N]], [[TMP34]]
; IC5-NEXT: br i1 [[TMP35]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC5: [[SCALAR_PH]]:
-; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP32]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC5-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC5-NEXT: br label %[[LOOP:.*]]
; IC5: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8f2e0297aab7d8..331bef81f46242 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -60,8 +60,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP15]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -140,29 +140,28 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
+; CHECK-NEXT: [[TMP13:%.*]] = freeze i1 [[TMP19]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
-; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP21]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[TMP28:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX]])
; CHECK-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP25]], <4 x float> [[TMP26]])
; CHECK-NEXT: [[TMP29:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX9]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP13]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -313,8 +312,8 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP52]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP50]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP55:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP50]]
-; CHECK-NEXT: [[TMP56:%.*]] = freeze <4 x i1> [[TMP55]]
-; CHECK-NEXT: [[TMP57:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP56]])
+; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
+; CHECK-NEXT: [[TMP57:%.*]] = freeze i1 [[TMP56]]
; CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP59:%.*]] = or i1 [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index b0dd95735c5f35..be0b092af95610 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -206,8 +206,8 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -271,8 +271,8 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -338,8 +338,8 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -406,8 +406,8 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -698,8 +698,8 @@ define float @test_fmax_and_fmax(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -817,8 +817,8 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 {
; CHECK-NEXT: [[TMP24]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP23]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]]
-; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP26]])
+; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
+; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP26]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 24999536d936c4..c4d12731434eea 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -206,8 +206,8 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -271,8 +271,8 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
index 9a6e990b7a84cc..967c70f0943465 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -373,10 +373,10 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD8]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP33]]
-; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = or <4 x i1> [[TMP5]], [[TMP16]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP37]]
-; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP17]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP5]], [[TMP16]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP38]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP19]], [[TMP20]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]])
; CHECK-VF4IC4-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -470,10 +470,10 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp sgt i32 [[TMP22]], 3
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[CMP1]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
-; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP49]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[CMP1]], i1 [[TMP3]]
; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP4]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index 58ffa4c050973e..ede9f3eb7ce04b 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -777,10 +777,10 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP33]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP34]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -868,10 +868,10 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
-; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
-; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP46:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP46]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
@@ -1269,10 +1269,10 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP29]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1335,10 +1335,10 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
-; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
-; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP43]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
@@ -1486,10 +1486,10 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP39:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP39]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1583,10 +1583,10 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
-; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
-; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP55:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP55]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
@@ -1979,10 +1979,10 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP39:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP39]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -2082,10 +2082,10 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP3]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
-; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
-; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP55:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP55]], [[TMP35]]
; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP3]], i1 [[TMP12]]
; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
@@ -2267,10 +2267,10 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[STEP_ADD_3]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP18]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP19]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; CHECK-VF4IC4-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> [[TMP2]]
@@ -2348,10 +2348,10 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP8]], 0
; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = freeze i1 [[TMP14]]
; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = freeze i1 [[TMP16]]
-; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]]
; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = freeze i1 [[TMP17]]
+; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP18]], [[TMP19]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = or i1 [[TMP38]], [[TMP21]]
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = or i1 [[TMP22]], [[TMP23]]
; CHECK-VF1IC4-NEXT: [[TMP25]] = select i1 [[TMP24]], i1 [[TMP14]], i1 [[TMP2]]
; CHECK-VF1IC4-NEXT: [[TMP26]] = select i1 [[TMP24]], i1 [[TMP15]], i1 [[TMP3]]
@@ -2507,10 +2507,10 @@ define i32 @select_icmp_truncated_iv_also_widened(ptr noalias %a, ptr noalias %b
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = freeze <4 x i1> [[TMP6]]
-; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP29]], [[TMP30]]
; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = freeze <4 x i1> [[TMP7]]
-; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP8]]
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP29]], [[TMP30]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP33]], [[TMP18]]
; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]])
; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP20]], <4 x i1> [[TMP5]], <4 x i1> [[TMP9]]
@@ -2603,10 +2603,10 @@ define i32 @select_icmp_truncated_iv_also_widened(ptr noalias %a, ptr noalias %b
; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = icmp sgt i32 [[TMP12]], 3
; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = freeze i1 [[TMP13]]
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = freeze i1 [[TMP14]]
-; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = or i1 [[TMP44]], [[TMP26]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = or i1 [[TMP44]], [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP49]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP31]], i1 [[TMP13]], i1 [[TMP1]]
; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP31]], i1 [[TMP14]], i1 [[TMP17]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
index 843b58d1d60127..efbbe480e0041c 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll
@@ -1969,10 +1969,10 @@ define float @not_vectorized_select_float_induction_icmp(ptr %a, ptr %b, float %
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = icmp sgt <4 x i64> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP15]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP16]]
-; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP41:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP41]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP22]], [[TMP23]]
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; CHECK-VF4IC4-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index 7969a080678eff..00bf9310e76076 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -1167,10 +1167,10 @@ define float @select_const_f32_from_icmp(ptr %v, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
-; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP3]], [[TMP13]]
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
-; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP3]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP33]], [[TMP15]]
; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
@@ -1245,10 +1245,10 @@ define float @select_const_f32_from_icmp(ptr %v, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = icmp ne i32 [[TMP14]], 3
; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP15]]
; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = freeze i1 [[TMP16]]
-; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = or i1 [[TMP19]], [[TMP20]]
; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = freeze i1 [[TMP17]]
-; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = or i1 [[TMP21]], [[TMP22]]
; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[TMP18]]
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = or i1 [[TMP19]], [[TMP20]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = or i1 [[TMP39]], [[TMP22]]
; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = or i1 [[TMP23]], [[TMP24]]
; CHECK-VF1IC4-NEXT: [[TMP26]] = select i1 [[TMP25]], i1 [[TMP15]], i1 [[TMP0]]
; CHECK-VF1IC4-NEXT: [[TMP27]] = select i1 [[TMP25]], i1 [[TMP16]], i1 [[TMP1]]
@@ -1395,10 +1395,10 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP17]], [[TMP24]]
; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP14]]
-; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP33]]
; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = or <4 x i1> [[TMP17]], [[TMP24]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
@@ -1483,10 +1483,10 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp ne i32 [[TMP14]], 3
; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP23]]
; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
-; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
-; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP47]], [[TMP30]]
; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[TMP23]], i1 [[TMP0]]
; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
index ca061fc0ef6e20..64d5820f321299 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -19,9 +19,12 @@ define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[A]], align 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM:.*]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
index 082c2dcb5d6d4a..2ffafcb118f24a 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
@@ -27,10 +27,10 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
; VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP2]]
; VF4IC4-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP6]]
-; VF4IC4-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
; VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP7]]
-; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP10]], [[TMP15]]
; VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP8]]
+; VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
+; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP18]], [[TMP15]]
; VF4IC4-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
index 749c4d93613b63..776d02f0c70a71 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
@@ -27,10 +27,10 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
; VF4IC4-NEXT: [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
; VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP8]]
; VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP6]]
-; VF4IC4-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP17]], [[TMP13]]
; VF4IC4-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP7]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP9]], [[TMP10]]
; VF4IC4-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP14]]
+; VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP17]], [[TMP13]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP18]], [[TMP10]]
; VF4IC4-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP15]], [[TMP16]]
; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -108,10 +108,10 @@ define i64 @same_exit_block_pre_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -194,10 +194,10 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
; VF4IC4-NEXT: [[TMP29:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], splat (i8 72)
; VF4IC4-NEXT: [[TMP35:%.*]] = freeze <4 x i1> [[TMP17]]
; VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP14]]
-; VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP35]], [[TMP13]]
; VF4IC4-NEXT: [[TMP32:%.*]] = freeze <4 x i1> [[TMP28]]
-; VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
; VF4IC4-NEXT: [[TMP34:%.*]] = freeze <4 x i1> [[TMP29]]
+; VF4IC4-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP35]], [[TMP13]]
+; VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP12]], [[TMP32]]
; VF4IC4-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP33]], [[TMP34]]
; VF4IC4-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP9]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -286,10 +286,10 @@ define i64 @same_exit_block_post_inc_use() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -382,10 +382,10 @@ define i64 @diff_exit_block_pre_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -483,10 +483,10 @@ define i64 @diff_exit_block_post_inc_use1() {
; VF4IC4-NEXT: [[TMP35:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
; VF4IC4-NEXT: [[TMP37:%.*]] = freeze <4 x i1> [[TMP13]]
; VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP12]]
-; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[TMP35]]
+; VF4IC4-NEXT: [[TMP34:%.*]] = or <4 x i1> [[TMP37]], [[TMP33]]
+; VF4IC4-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP34]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP36]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -594,10 +594,10 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
; VF4IC4-NEXT: [[TMP43:%.*]] = icmp ne <4 x i8> [[REVERSE6]], [[REVERSE16]]
; VF4IC4-NEXT: [[TMP48:%.*]] = freeze <4 x i1> [[TMP21]]
; VF4IC4-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP19]]
-; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP48]], [[TMP22]]
; VF4IC4-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[TMP20]]
-; VF4IC4-NEXT: [[TMP45:%.*]] = or <4 x i1> [[TMP23]], [[TMP44]]
; VF4IC4-NEXT: [[TMP47:%.*]] = freeze <4 x i1> [[TMP43]]
+; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP48]], [[TMP22]]
+; VF4IC4-NEXT: [[TMP45:%.*]] = or <4 x i1> [[TMP23]], [[TMP44]]
; VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP45]], [[TMP47]]
; VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -704,10 +704,10 @@ define i8 @same_exit_block_use_loaded_value() {
; VF4IC4-NEXT: [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
; VF4IC4-NEXT: [[TMP43:%.*]] = freeze <4 x i1> [[TMP12]]
; VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP29]]
-; VF4IC4-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP43]], [[TMP18]]
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP11]]
-; VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP13]], [[TMP14]]
; VF4IC4-NEXT: [[TMP30:%.*]] = freeze <4 x i1> [[TMP17]]
+; VF4IC4-NEXT: [[TMP44:%.*]] = or <4 x i1> [[TMP43]], [[TMP18]]
+; VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP44]], [[TMP14]]
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP19]], [[TMP30]]
; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -822,10 +822,10 @@ define i8 @same_exit_block_reverse_use_loaded_value() {
; VF4IC4-NEXT: [[TMP37:%.*]] = icmp ne <4 x i8> [[REVERSE7]], [[REVERSE15]]
; VF4IC4-NEXT: [[TMP54:%.*]] = freeze <4 x i1> [[TMP21]]
; VF4IC4-NEXT: [[TMP22:%.*]] = freeze <4 x i1> [[TMP19]]
-; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP54]], [[TMP22]]
; VF4IC4-NEXT: [[TMP51:%.*]] = freeze <4 x i1> [[TMP20]]
-; VF4IC4-NEXT: [[TMP52:%.*]] = or <4 x i1> [[TMP23]], [[TMP51]]
; VF4IC4-NEXT: [[TMP53:%.*]] = freeze <4 x i1> [[TMP37]]
+; VF4IC4-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP54]], [[TMP22]]
+; VF4IC4-NEXT: [[TMP52:%.*]] = or <4 x i1> [[TMP23]], [[TMP51]]
; VF4IC4-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP52]], [[TMP53]]
; VF4IC4-NEXT: [[TMP25:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
>From 5c58636b2edcca2fae818b0ef2ba57b0679f2e38 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 18:00:30 +0800
Subject: [PATCH 3/9] Remove FIXME
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 2 +-
.../LoopVectorize/single-early-exit-anyof-fold.ll | 9 +++------
2 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4b1ebed273dcff..3adca452c117a1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1695,7 +1695,7 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
}
VPValue *AnyNaNLane =
- LatchBuilder.createNaryOp(VPInstruction::AnyOf, AllNaNLanes);
+ LatchBuilder.createNaryOp(VPInstruction::AnyOf, {AllNaNLanes});
// Freeze to prevent immediate UB from branching on poison.
AnyNaNLane = LatchBuilder.createFreeze(AnyNaNLane);
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
index 64d5820f321299..68516d230711e2 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -1,12 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
-; FIXME; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
-; replaces two individual freezes with one joint freeze which is incorrect.
-;
-; e.g. when x[0] = NaN and x[4] = poison the scalar loop is well defined and
-; returns true. When vectorized we get reduce.or(freeze(fcmp uno x[0]=NaN, x[4]=poison))
-; which may return false.
+; Make sure we don't carry out the
+; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
+; combine, since the freezes should block it.
define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
; CHECK-LABEL: define i1 @find_nan(
>From 49a71fa40aca5ad7b4bb5f02b4d64d47b6f511f1 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 18:49:29 +0800
Subject: [PATCH 4/9] Remove VPInstruction::usesFirstLaneOnly change
InstCombine will canonicalize this anyway https://godbolt.org/z/r7PsGxh6G
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +-
.../AArch64/fmax-without-fast-math-flags.ll | 62 ++++++++----
.../AArch64/fmin-without-fast-math-flags.ll | 25 +++--
.../LoopVectorize/X86/cost-any-of.ll | 22 +++--
...epilog-vectorization-fmaxnum-reductions.ll | 48 +++++++---
.../LoopVectorize/fcmp-uno-fold-interleave.ll | 75 ++++++++++-----
...fmax-without-fast-math-flags-interleave.ll | 63 ++++++++----
.../fmax-without-fast-math-flags.ll | 95 +++++++++++++------
.../fmin-without-fast-math-flags.ll | 32 +++++--
9 files changed, 290 insertions(+), 140 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4f3699e7cb8028..1dcdfec52fb077 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1741,15 +1741,9 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return Op == getOperand(1) || Op == getOperand(2);
case Instruction::PHI:
return true;
- case Instruction::Select:
- if (Op == getOperand(0) && Op != getOperand(1) && Op != getOperand(2) &&
- vputils::isSingleScalar(Op))
- // TODO: Relax for other opcodes.
- if (match(Op, m_Freeze(m_VPValue())))
- return true;
- [[fallthrough]];
case Instruction::FCmp:
case Instruction::ICmp:
+ case Instruction::Select:
case Instruction::Or:
case Instruction::Freeze:
case VPInstruction::Not:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 715f1a62252b38..c7f4e8fef7639e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -71,26 +71,35 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
-; CHECK-NEXT: [[TMP14:%.*]] = freeze i1 [[TMP6]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP14]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP29]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP28]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -177,20 +186,32 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP35]], [[TMP29]]
; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP34]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP42]])
-; CHECK-NEXT: [[TMP41:%.*]] = freeze i1 [[TMP19]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP19]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
; CHECK-NEXT: [[TMP27:%.*]] = or i1 [[TMP41]], [[TMP21]]
; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
-; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
-; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP41]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP43:%.*]] = select i1 [[TMP41]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP47]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP48]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP50]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
+; CHECK-NEXT: [[TMP51:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP51]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP52:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP52]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP53]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP54]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP43]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP46:%.*]] = select i1 [[TMP45]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP32]])
; CHECK-NEXT: [[RDX_MINMAX16:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX15]], <4 x float> [[TMP33]])
@@ -200,11 +221,12 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[RDX_MINMAX19:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX18]], <4 x float> [[TMP37]])
; CHECK-NEXT: [[TMP40:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX19]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP41]], true
+; CHECK-NEXT: [[TMP49:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP49]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP46]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi float [ [[TMP40]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 1221eac37d4d6a..6185bbd4feac7a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -71,26 +71,35 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
-; CHECK-NEXT: [[TMP14:%.*]] = freeze i1 [[TMP6]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP14]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP29]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP28]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
index 88abf190fafbb2..bf0a4de1624e86 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
@@ -34,16 +34,24 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP12]])
-; CHECK-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP13]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TMP13]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <16 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP15]], [[TMP14]]
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP15]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 128
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP26]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP27]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP28]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP22]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP24]], i64 [[INDEX]], i64 128
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[TMP16]], <16 x float> [[TMP17]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX]], <16 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX7]], <16 x float> [[TMP19]])
@@ -52,7 +60,7 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP22]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP25]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RED:%.*]] = phi float [ [[TMP21]], %[[SCALAR_PH]] ], [ [[MIN:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LDV:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
index a5a4cf265a53c3..7cef0da0647a36 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
@@ -22,20 +22,26 @@ define float @fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -87,20 +93,26 @@ define float @fminnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 9.900000e+02, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -152,20 +164,26 @@ define i64 @dead_fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
index 19e499443e0f51..035d1366d80d7a 100644
--- a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
@@ -32,24 +32,32 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC3-NEXT: [[TMP7:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
; IC3-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
-; IC3-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP14]]
+; IC3-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP14]], i64 0
+; IC3-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; IC3-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC3-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC3-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
; IC3-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP15]]
; IC3-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC3: [[MIDDLE_BLOCK]]:
-; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
-; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
-; IC3-NEXT: [[TMP16:%.*]] = select i1 [[TMP10]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC3-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
+; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; IC3-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
+; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
+; IC3-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
+; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP25]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
+; IC3-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
+; IC3-NEXT: [[TMP26:%.*]] = select i1 [[TMP20]], i64 [[INDEX]], i64 [[N_VEC]]
; IC3-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP17]], <4 x float> [[TMP18]])
; IC3-NEXT: [[RDX_MINMAX5:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP19]])
; IC3-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX5]])
; IC3-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP10]], true
+; IC3-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
+; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP27]], true
; IC3-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC3-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC3: [[SCALAR_PH]]:
-; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC3-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC3-NEXT: br label %[[LOOP:.*]]
; IC3: [[LOOP]]:
@@ -86,26 +94,35 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC4-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP24]], [[TMP25]]
; IC4-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP29]])
-; IC4-NEXT: [[TMP26:%.*]] = freeze i1 [[TMP13]]
+; IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP13]], i64 0
+; IC4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; IC4-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC4-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
; IC4-NEXT: [[TMP15:%.*]] = or i1 [[TMP26]], [[TMP14]]
; IC4-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC4: [[MIDDLE_BLOCK]]:
-; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
-; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
-; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
-; IC4-NEXT: [[TMP20:%.*]] = select i1 [[TMP26]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC4-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; IC4-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
+; IC4-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
+; IC4-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
+; IC4-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP34:%.*]] = select i1 [[TMP33]], i64 [[INDEX]], i64 [[N_VEC]]
; IC4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP16]], <4 x float> [[TMP17]])
; IC4-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP18]])
; IC4-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; IC4-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP26]], true
+; IC4-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
+; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP28]], true
; IC4-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC4-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC4: [[SCALAR_PH]]:
-; IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP34]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC4-NEXT: br label %[[LOOP:.*]]
; IC4: [[LOOP]]:
@@ -148,28 +165,38 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC5-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
; IC5-NEXT: [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP23]])
-; IC5-NEXT: [[TMP16:%.*]] = freeze i1 [[TMP24]]
+; IC5-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP24]], i64 0
+; IC5-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; IC5-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC5-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC5-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
; IC5-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP25]]
; IC5-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC5: [[MIDDLE_BLOCK]]:
-; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
-; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
-; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
-; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
-; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
-; IC5-NEXT: [[TMP26:%.*]] = select i1 [[TMP16]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC5-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
+; IC5-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP22]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
+; IC5-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
+; IC5-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
+; IC5-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP36]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
+; IC5-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i64 [[INDEX]], i64 [[N_VEC]]
; IC5-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP27]], <4 x float> [[TMP28]])
; IC5-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP29]])
; IC5-NEXT: [[RDX_MINMAX10:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX9]], <4 x float> [[TMP30]])
; IC5-NEXT: [[RDX_MINMAX11:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX10]], <4 x float> [[TMP31]])
; IC5-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX11]])
; IC5-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP16]], true
+; IC5-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
+; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP39]], true
; IC5-NEXT: [[TMP35:%.*]] = and i1 [[CMP_N]], [[TMP34]]
; IC5-NEXT: br i1 [[TMP35]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC5: [[SCALAR_PH]]:
-; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP38]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC5-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC5-NEXT: br label %[[LOOP:.*]]
; IC5: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 331bef81f46242..28c13666c8ccd1 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -61,22 +61,29 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP15]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP15]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP21]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP18]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -142,26 +149,35 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
-; CHECK-NEXT: [[TMP13:%.*]] = freeze i1 [[TMP19]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP19]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP21]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP17]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP22]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP32]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[TMP28:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX]])
; CHECK-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP25]], <4 x float> [[TMP26]])
; CHECK-NEXT: [[TMP29:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX9]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP34]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -313,25 +329,32 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP55:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP50]]
; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
-; CHECK-NEXT: [[TMP57:%.*]] = freeze i1 [[TMP56]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[TMP56]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP62:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT17]]
; CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP57:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
; CHECK-NEXT: [[TMP59:%.*]] = or i1 [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: br i1 [[TMP59]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP53:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP51]], <4 x float> [[VEC_PHI]]
; CHECK-NEXT: [[TMP54:%.*]] = select <4 x i1> [[TMP2]], <4 x float> [[TMP52]], <4 x float> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP53]]
-; CHECK-NEXT: [[TMP61:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP54]]
-; CHECK-NEXT: [[TMP62:%.*]] = select i1 [[TMP57]], i64 [[INDEX]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP67:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
+; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP67]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP53]]
+; CHECK-NEXT: [[TMP69:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
+; CHECK-NEXT: [[TMP61:%.*]] = select i1 [[TMP69]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP54]]
+; CHECK-NEXT: [[TMP65:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
+; CHECK-NEXT: [[TMP66:%.*]] = select i1 [[TMP65]], i64 [[INDEX]], i64 [[TMP0]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP60]], <4 x float> [[TMP61]])
; CHECK-NEXT: [[TMP63:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX]])
-; CHECK-NEXT: [[TMP64:%.*]] = xor i1 [[TMP57]], true
+; CHECK-NEXT: [[TMP68:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
+; CHECK-NEXT: [[TMP64:%.*]] = xor i1 [[TMP68]], true
; CHECK-NEXT: br i1 [[TMP64]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP62]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP66]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[TMP63]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index be0b092af95610..8c5719996f6f02 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -207,20 +207,26 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -272,20 +278,26 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -339,17 +351,23 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP6]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP18]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = add i64 10, [[TMP9]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP17]], true
; CHECK-NEXT: [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -407,17 +425,23 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP12]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP6]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP18]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[START]], [[TMP9]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP17]], true
; CHECK-NEXT: [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -699,22 +723,29 @@ define float @test_fmax_and_fmax(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP6]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP2]]
-; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP9]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP16:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP13]])
; CHECK-NEXT: [[TMP17:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP14]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP9]], true
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP22]], true
; CHECK-NEXT: [[TMP19:%.*]] = and i1 [[CMP_N]], [[TMP18]]
; CHECK-NEXT: br i1 [[TMP19]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi float [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -818,22 +849,28 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]]
; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP26]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP0]], <4 x float> [[TMP24]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP27]], i64 [[INDEX]], i64 30
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
+; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]]
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = select i1 [[TMP38]], i64 [[INDEX]], i64 30
; CHECK-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP27]], true
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
+; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP37]], true
; CHECK-NEXT: br i1 [[TMP34]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP32]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP35]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[TMP33]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index c4d12731434eea..ae5228d9431fba 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -207,20 +207,26 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -272,20 +278,26 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP10]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
>From 87ea68e9e47e37e86ffef13fcbb993795a3ff878 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 19:15:53 +0800
Subject: [PATCH 5/9] Don't unwrap the freeze, copy it over in side effect
early exit loops
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 ++---
.../AArch64/early_exit_with_stores.ll | 16 +++---
.../RISCV/early_exit_with_stores.ll | 16 +++---
.../VPlan/early_exit_with_stores_vplan.ll | 54 +++++++++----------
.../LoopVectorize/early_exit_with_stores.ll | 24 ++++-----
.../interleave_uncountable_exits.ll | 4 +-
...rized_conditional_ops_uncountable_exits.ll | 4 +-
.../tail_fold_uncountable_exits.ll | 4 +-
8 files changed, 67 insertions(+), 67 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 65406fd8f74e85..8dd3977d1d328f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3086,10 +3086,9 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// Find the uncountable loop exit condition.
VPValue *UncountableCondition = nullptr;
- if (!match(
- LatchVPBB->getTerminator(),
- m_BranchOnTwoConds(m_AnyOf(m_Freeze(m_VPValue(UncountableCondition))),
- m_VPValue())))
+ if (!match(LatchVPBB->getTerminator(),
+ m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
+ m_VPValue())))
return nullptr;
SmallVector<VPValue *, 4> Worklist;
@@ -3124,8 +3123,9 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
return nullptr;
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
Recipes.push_back(cast<VPInstruction>(GepR));
- } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op1)))) {
+ } else if (match(V, m_CombineOr(m_VPInstruction<VPInstruction::MaskedCond>(
+ m_VPValue(Op1)),
+ m_Freeze(m_VPValue(Op1))))) {
Worklist.push_back(Op1);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
} else
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 38c8133ddc8d14..faba88cee2203f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -66,13 +66,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -344,14 +344,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -468,13 +468,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR1]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1140,13 +1140,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 8e71583ef56af5..beee8ddce5f130 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -64,13 +64,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -342,14 +342,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -511,13 +511,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1183,13 +1183,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 2fb9d906e22cb2..fdc571c281c02f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -27,16 +27,16 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
@@ -46,7 +46,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -117,16 +117,16 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
@@ -136,7 +136,7 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -203,16 +203,16 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP9]]>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
@@ -224,7 +224,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part ir<%data>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 7e9a73d197a10c..bdd795ac02cf4a 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -59,13 +59,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -331,14 +331,14 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP27]], ptr align 2 [[TMP29]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -449,13 +449,13 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1025,13 +1025,13 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1187,13 +1187,13 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
@@ -1330,13 +1330,13 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i16> splat (i16 500), [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index d1ef3a607a243b..8637897644caee 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -15,13 +15,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index c572f150860828..4523ba2d7d3e7b 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -16,7 +16,8 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
@@ -79,7 +80,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: store i16 [[TMP30]], ptr [[TMP34]], align 2
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index 58ff87764459a6..e4633bcb5568a0 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -18,13 +18,13 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
>From b6bcf4290bafa401c0f030bc8573c4419e5c8e26 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 15 Sep 2026 19:24:52 +0800
Subject: [PATCH 6/9] Update comments
---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d570002fb82367..4534ff881fcadb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1378,7 +1378,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Returns a scalar boolean value, which is true if any lane of its
// (boolean) vector operands is true. It produces the reduced value across
// all unrolled iterations. Unrolling will add all copies of its original
- // operand as additional operands.
+ // operand as additional operands. Note does not block poison propagation.
AnyOf,
// Calculates the first active lane index of the vector predicate operands.
// It produces the lane index across all unrolled iterations. Unrolling will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8dd3977d1d328f..e4897afcc53c2c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3168,7 +3168,8 @@ struct EarlyExitInfo {
/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
/// EMIT store ir<%add>, ir<%arrayidx5>
/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
-/// EMIT vp<%3> = any-of ir<%1>
+/// EMIT vp<%freeze> = freeze ir<%1>
+/// EMIT vp<%3> = any-of ir<%freeze>
/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
/// Successor(s): middle.block, middle.block, for.body
>From 368e8958bbdee8dca12ce22cd915421603d8fac9 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 16 Sep 2026 22:39:27 +0800
Subject: [PATCH 7/9] Update comment
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e4897afcc53c2c..6ded858805e77f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3409,8 +3409,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(
for (const EarlyExitInfo &Info : drop_begin(Exits))
Combined = LatchBuilder.createLogicalOr(Combined, Info.CondToExit);
- // Even though the logical or prevents per-lane posion propagation, we need to
- // freeze Combined to prevent poisoning the entire AnyOf result:
+ // Even though the logical or prevents posion propagation, we need to freeze
+ // Combined to prevent poisoning the entire AnyOf result:
//
// Exits[0].CondToExit = [0,1,0,0]
// Exits[1].CondToExit = [0,0,p,p]
>From 82b6b78c860af16daeb50fb55b60c101f32d98fe Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 17 Sep 2026 20:39:18 +0800
Subject: [PATCH 8/9] Freeze MinOrMaxOp earlier
---
.../Vectorize/VPlanConstruction.cpp | 13 +-
.../LoopVectorize/AArch64/cmp_cost.ll | 24 +--
.../AArch64/fmax-without-fast-math-flags.ll | 124 +++++++---------
.../AArch64/fmin-without-fast-math-flags.ll | 46 +++---
.../sve-non-native-interleaved-accesses.ll | 19 ++-
.../LoopVectorize/X86/cost-any-of.ll | 43 +++---
...epilog-vectorization-fmaxnum-reductions.ll | 66 +++------
.../LoopVectorize/fcmp-uno-fold-interleave.ll | 140 ++++++++----------
...fmax-without-fast-math-flags-interleave.ll | 100 +++++--------
.../fmax-without-fast-math-flags.ll | 132 ++++++-----------
.../fmin-without-fast-math-flags.ll | 44 ++----
.../LoopVectorize/minmax_reduction.ll | 14 +-
.../single-early-exit-anyof-fold.ll | 9 +-
13 files changed, 335 insertions(+), 439 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 3adca452c117a1..ba4d14ec3a111f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1683,6 +1683,17 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
}
}
+ // Freeze MinOrMaxOps since:
+ // * multiple uses are introduced and the value may be undef
+ // * they feed into a branch condition, so block poison propagation to
+ // prevent immediate UB
+ for (auto &[Phi, MinOrMaxOp] : MinOrMaxNumReductionsToHandle) {
+ VPRecipeBase *MinOrMaxR = Phi->getBackedgeValue()->getDefiningRecipe();
+ VPInstruction *Freeze = VPBuilder(MinOrMaxR).createFreeze(MinOrMaxOp);
+ MinOrMaxR->replaceUsesOfWith(MinOrMaxOp, Freeze);
+ MinOrMaxOp = Freeze;
+ }
+
VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
VPValue *AllNaNLanes = nullptr;
@@ -1696,8 +1707,6 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
VPValue *AnyNaNLane =
LatchBuilder.createNaryOp(VPInstruction::AnyOf, {AllNaNLanes});
- // Freeze to prevent immediate UB from branching on poison.
- AnyNaNLane = LatchBuilder.createFreeze(AnyNaNLane);
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->begin());
for (const auto &[RedPhiR, _] : MinOrMaxNumReductionsToHandle) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index fee76962809427..83ec5cb9c9b574 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -15,11 +15,11 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = any-of vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
@@ -46,11 +46,11 @@ define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 4: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP4]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP4]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP5]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP6]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP5]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP6]]> = any-of vp<[[VP5]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP8]]> = or vp<[[VP6]]>, vp<[[VP7]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP8]]>
@@ -98,11 +98,11 @@ define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to double
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
-; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
+; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = freeze ir<%v>
+; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, vp<[[VP4]]>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
-; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
-; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
-; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = freeze vp<[[VP5]]>
+; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = fcmp uno vp<[[VP4]]>, vp<[[VP4]]>
+; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = any-of vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP7]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP8]]>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index c7f4e8fef7639e..7aff41be08a001 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -62,44 +62,38 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP25]])
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP26]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP27]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP22]], [[TMP25]]
+; CHECK-NEXT: [[TMP23:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP27]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP29]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP28]], true
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -169,49 +163,44 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP22]], align 4
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP28]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP10]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI6]], <4 x float> [[WIDE_LOAD9]])
-; CHECK-NEXT: [[TMP11]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI7]], <4 x float> [[WIDE_LOAD10]])
-; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
-; CHECK-NEXT: [[TMP14]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[WIDE_LOAD13]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI5]], <4 x float> [[WIDE_LOAD14]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP29:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP34:%.*]] = freeze <4 x float> [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP43:%.*]] = freeze <4 x float> [[WIDE_LOAD10]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP29]])
+; CHECK-NEXT: [[TMP10]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI6]], <4 x float> [[TMP34]])
+; CHECK-NEXT: [[TMP11]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI7]], <4 x float> [[TMP43]])
+; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP45:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP46:%.*]] = freeze <4 x float> [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP47:%.*]] = freeze <4 x float> [[WIDE_LOAD14]]
+; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP44]])
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP45]])
+; CHECK-NEXT: [[TMP14]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[TMP46]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI5]], <4 x float> [[TMP47]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP29:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD13]], [[WIDE_LOAD9]]
-; CHECK-NEXT: [[TMP34:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD14]], [[WIDE_LOAD10]]
-; CHECK-NEXT: [[TMP35:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP35]], [[TMP29]]
-; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP34]]
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp uno <4 x float> [[TMP44]], [[TMP9]]
+; CHECK-NEXT: [[TMP17:%.*]] = fcmp uno <4 x float> [[TMP45]], [[TMP29]]
+; CHECK-NEXT: [[TMP35:%.*]] = fcmp uno <4 x float> [[TMP46]], [[TMP34]]
+; CHECK-NEXT: [[TMP41:%.*]] = fcmp uno <4 x float> [[TMP47]], [[TMP43]]
+; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP38:%.*]] = or <4 x i1> [[TMP18]], [[TMP35]]
+; CHECK-NEXT: [[TMP42:%.*]] = or <4 x i1> [[TMP38]], [[TMP41]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP42]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP19]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP27:%.*]] = or i1 [[TMP41]], [[TMP21]]
-; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP47]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP48]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP50]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
-; CHECK-NEXT: [[TMP51:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP51]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP52:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP52]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP53:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP53]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP54]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
-; CHECK-NEXT: [[TMP43:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP43]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP46:%.*]] = select i1 [[TMP45]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]]
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI5]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP36:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI6]], <4 x float> [[TMP10]]
+; CHECK-NEXT: [[TMP37:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI7]], <4 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[RDX_MINMAX15:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP32]])
; CHECK-NEXT: [[RDX_MINMAX16:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX15]], <4 x float> [[TMP33]])
@@ -221,12 +210,11 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[RDX_MINMAX19:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX18]], <4 x float> [[TMP37]])
; CHECK-NEXT: [[TMP40:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX19]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP49:%.*]] = extractelement <4 x i1> [[TMP44]], i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP49]], true
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP46]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP39]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi float [ [[TMP40]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 6185bbd4feac7a..4cecec4d4663db 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -62,44 +62,38 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP25]])
+; CHECK-NEXT: [[TMP13]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP26]])
+; CHECK-NEXT: [[TMP15]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP27]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP4]], [[TMP22]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP22]], [[TMP25]]
+; CHECK-NEXT: [[TMP23:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP27]]
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP5]], [[TMP23]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP27:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP14]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP29]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP13]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX8]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP28]], true
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 624e904fbfbc08..93c3152cbcc415 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -40,15 +40,18 @@ define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0)
; CHECK-NEXT: [[TMP18:%.*]] = fmul float 0.000000e+00, [[TMP59]]
; CHECK-NEXT: [[TMP19:%.*]] = fmul float 0.000000e+00, [[TMP60]]
; CHECK-NEXT: [[TMP20:%.*]] = fmul float 0.000000e+00, [[TMP61]]
-; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP17]])
-; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP18]])
-; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP19]])
-; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP20]])
+; CHECK-NEXT: [[TMP25:%.*]] = freeze float [[TMP17]]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze float [[TMP18]]
+; CHECK-NEXT: [[TMP29:%.*]] = freeze float [[TMP19]]
+; CHECK-NEXT: [[TMP44:%.*]] = freeze float [[TMP20]]
+; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP25]])
+; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP26]])
+; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP29]])
+; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP44]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
-; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP20]]
-; CHECK-NEXT: [[TMP27:%.*]] = or i1 [[TMP25]], [[TMP26]]
-; CHECK-NEXT: [[TMP76:%.*]] = freeze i1 [[TMP27]]
+; CHECK-NEXT: [[TMP27:%.*]] = fcmp uno float [[TMP25]], [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = fcmp uno float [[TMP29]], [[TMP44]]
+; CHECK-NEXT: [[TMP76:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP78:%.*]] = or i1 [[TMP76]], [[TMP77]]
; CHECK-NEXT: br i1 [[TMP78]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
index bf0a4de1624e86..277b75c867ca15 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-any-of.ll
@@ -25,33 +25,28 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI]], <16 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI1]], <16 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP6]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI2]], <16 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI3]], <16 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <16 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <16 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP22:%.*]] = freeze <16 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <16 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP4]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI]], <16 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP5]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI1]], <16 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP6]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI2]], <16 x float> [[TMP22]])
+; CHECK-NEXT: [[TMP7]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[VEC_PHI3]], <16 x float> [[TMP23]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <16 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <16 x float> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fcmp uno <16 x float> [[TMP22]], [[TMP23]]
+; CHECK-NEXT: [[TMP12:%.*]] = or <16 x i1> [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP12]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TMP13]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = freeze <16 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP15]], [[TMP14]]
-; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP14]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP26]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP27]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP28]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP22]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <16 x i1> [[TMP23]], i64 0
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP24]], i64 [[INDEX]], i64 128
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI]], <16 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI1]], <16 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI2]], <16 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP13]], <16 x float> [[VEC_PHI3]], <16 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[INDEX]], i64 128
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[TMP16]], <16 x float> [[TMP17]])
; CHECK-NEXT: [[RDX_MINMAX7:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX]], <16 x float> [[TMP18]])
; CHECK-NEXT: [[RDX_MINMAX8:%.*]] = call <16 x float> @llvm.minnum.v16f32(<16 x float> [[RDX_MINMAX7]], <16 x float> [[TMP19]])
@@ -60,7 +55,7 @@ define void @fminnum_with_any_of_cost(ptr %p) #0 {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP25]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RED:%.*]] = phi float [ [[TMP21]], %[[SCALAR_PH]] ], [ [[MIN:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[LDV:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
index 7cef0da0647a36..1ceb488c176831 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
@@ -18,30 +18,24 @@ define float @fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -89,30 +83,24 @@ define float @fminnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float 9.900000e+02), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 9.900000e+02, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -160,30 +148,24 @@ define i64 @dead_fmaxnum_reduction(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[TMP12]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <8 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP13]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP4]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP4]], true
; CHECK-NEXT: [[TMP11:%.*]] = and i1 [[CMP_N]], [[TMP10]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
index 035d1366d80d7a..3e1a8b68bb2aa1 100644
--- a/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
@@ -24,40 +24,34 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC3-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
; IC3-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; IC3-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; IC3-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC3-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD3]])
-; IC3-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD4]])
+; IC3-NEXT: [[TMP6:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC3-NEXT: [[TMP7:%.*]] = freeze <4 x float> [[WIDE_LOAD3]]
+; IC3-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; IC3-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP6]])
+; IC3-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]])
+; IC3-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP8]])
; IC3-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 12
-; IC3-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
-; IC3-NEXT: [[TMP7:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
+; IC3-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP8]], [[TMP8]]
+; IC3-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP6]], [[TMP7]]
+; IC3-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
; IC3-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
-; IC3-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP14]], i64 0
-; IC3-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; IC3-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC3-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC3-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP15]]
-; IC3-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC3-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]]
+; IC3-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC3: [[MIDDLE_BLOCK]]:
-; IC3-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; IC3-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
-; IC3-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP25]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
-; IC3-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP26:%.*]] = select i1 [[TMP20]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC3-NEXT: [[TMP17:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; IC3-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]]
+; IC3-NEXT: [[TMP19:%.*]] = select i1 [[TMP14]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP5]]
+; IC3-NEXT: [[TMP20:%.*]] = select i1 [[TMP14]], i64 [[INDEX]], i64 [[N_VEC]]
; IC3-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP17]], <4 x float> [[TMP18]])
; IC3-NEXT: [[RDX_MINMAX5:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP19]])
; IC3-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX5]])
; IC3-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC3-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
-; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP27]], true
+; IC3-NEXT: [[TMP22:%.*]] = xor i1 [[TMP14]], true
; IC3-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC3-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC3: [[SCALAR_PH]]:
-; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP26]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC3-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC3-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC3-NEXT: br label %[[LOOP:.*]]
; IC3: [[LOOP]]:
@@ -85,44 +79,38 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; IC4-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC4-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD4]])
-; IC4-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD5]])
-; IC4-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD6]])
+; IC4-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC4-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; IC4-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; IC4-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; IC4-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP9]])
+; IC4-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP10]])
+; IC4-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP11]])
+; IC4-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]])
; IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; IC4-NEXT: [[TMP24:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; IC4-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
-; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP24]], [[TMP25]]
+; IC4-NEXT: [[TMP26:%.*]] = fcmp uno <4 x float> [[TMP9]], [[TMP10]]
+; IC4-NEXT: [[TMP28:%.*]] = fcmp uno <4 x float> [[TMP11]], [[TMP8]]
+; IC4-NEXT: [[TMP29:%.*]] = or <4 x i1> [[TMP26]], [[TMP28]]
; IC4-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP29]])
-; IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP13]], i64 0
-; IC4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; IC4-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC4-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP15:%.*]] = or i1 [[TMP26]], [[TMP14]]
+; IC4-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP14]]
; IC4-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC4: [[MIDDLE_BLOCK]]:
-; IC4-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; IC4-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP30]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
-; IC4-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP31]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
-; IC4-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
-; IC4-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP34:%.*]] = select i1 [[TMP33]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC4-NEXT: [[TMP16:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; IC4-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP5]]
+; IC4-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP6]]
+; IC4-NEXT: [[TMP19:%.*]] = select i1 [[TMP13]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP7]]
+; IC4-NEXT: [[TMP20:%.*]] = select i1 [[TMP13]], i64 [[INDEX]], i64 [[N_VEC]]
; IC4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP16]], <4 x float> [[TMP17]])
; IC4-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP18]])
; IC4-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX7]], <4 x float> [[TMP19]])
; IC4-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX8]])
; IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC4-NEXT: [[TMP28:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
-; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP28]], true
+; IC4-NEXT: [[TMP22:%.*]] = xor i1 [[TMP13]], true
; IC4-NEXT: [[TMP23:%.*]] = and i1 [[CMP_N]], [[TMP22]]
; IC4-NEXT: br i1 [[TMP23]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC4: [[SCALAR_PH]]:
-; IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP34]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC4-NEXT: br label %[[LOOP:.*]]
; IC4: [[LOOP]]:
@@ -153,50 +141,44 @@ define float @fmaxnum(ptr %src, i64 %n) {
; IC5-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; IC5-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; IC5-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
-; IC5-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; IC5-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD5]])
-; IC5-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD6]])
-; IC5-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD7]])
-; IC5-NEXT: [[TMP9]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[WIDE_LOAD8]])
+; IC5-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; IC5-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; IC5-NEXT: [[TMP12:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; IC5-NEXT: [[TMP13:%.*]] = freeze <4 x float> [[WIDE_LOAD7]]
+; IC5-NEXT: [[TMP14:%.*]] = freeze <4 x float> [[WIDE_LOAD8]]
+; IC5-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP10]])
+; IC5-NEXT: [[TMP6]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP11]])
+; IC5-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP12]])
+; IC5-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP13]])
+; IC5-NEXT: [[TMP9]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI4]], <4 x float> [[TMP14]])
; IC5-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 20
-; IC5-NEXT: [[TMP14:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD8]], [[WIDE_LOAD8]]
-; IC5-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD5]]
-; IC5-NEXT: [[TMP13:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD7]]
-; IC5-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
-; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP15]], [[TMP14]]
+; IC5-NEXT: [[TMP22:%.*]] = fcmp uno <4 x float> [[TMP14]], [[TMP14]]
+; IC5-NEXT: [[TMP18:%.*]] = fcmp uno <4 x float> [[TMP10]], [[TMP11]]
+; IC5-NEXT: [[TMP20:%.*]] = fcmp uno <4 x float> [[TMP12]], [[TMP13]]
+; IC5-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP18]], [[TMP20]]
+; IC5-NEXT: [[TMP23:%.*]] = or <4 x i1> [[TMP21]], [[TMP22]]
; IC5-NEXT: [[TMP24:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP23]])
-; IC5-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP24]], i64 0
-; IC5-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; IC5-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; IC5-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; IC5-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP25]]
-; IC5-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC5-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; IC5-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; IC5: [[MIDDLE_BLOCK]]:
-; IC5-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
-; IC5-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP22]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
-; IC5-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
-; IC5-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP26]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
-; IC5-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP36]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
-; IC5-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP38:%.*]] = select i1 [[TMP37]], i64 [[INDEX]], i64 [[N_VEC]]
+; IC5-NEXT: [[TMP27:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP5]]
+; IC5-NEXT: [[TMP28:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP6]]
+; IC5-NEXT: [[TMP29:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP7]]
+; IC5-NEXT: [[TMP30:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP8]]
+; IC5-NEXT: [[TMP31:%.*]] = select i1 [[TMP24]], <4 x float> [[VEC_PHI4]], <4 x float> [[TMP9]]
+; IC5-NEXT: [[TMP32:%.*]] = select i1 [[TMP24]], i64 [[INDEX]], i64 [[N_VEC]]
; IC5-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP27]], <4 x float> [[TMP28]])
; IC5-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX]], <4 x float> [[TMP29]])
; IC5-NEXT: [[RDX_MINMAX10:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX9]], <4 x float> [[TMP30]])
; IC5-NEXT: [[RDX_MINMAX11:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[RDX_MINMAX10]], <4 x float> [[TMP31]])
; IC5-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX11]])
; IC5-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; IC5-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP17]], i64 0
-; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP39]], true
+; IC5-NEXT: [[TMP34:%.*]] = xor i1 [[TMP24]], true
; IC5-NEXT: [[TMP35:%.*]] = and i1 [[CMP_N]], [[TMP34]]
; IC5-NEXT: br i1 [[TMP35]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; IC5: [[SCALAR_PH]]:
-; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP38]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC5-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP32]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; IC5-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; IC5-NEXT: br label %[[LOOP:.*]]
; IC5: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 28c13666c8ccd1..9557c1167abfbf 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -56,34 +56,28 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP15]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP21]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP18]], true
+; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -140,44 +134,38 @@ define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i64 4
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
-; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x float> [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x float> [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x float> [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP9]])
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp uno <4 x float> [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP17:%.*]] = fcmp uno <4 x float> [[TMP10]], [[TMP11]]
+; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP19]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP21]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP17]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP22]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP32]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
+; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
; CHECK-NEXT: [[TMP28:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX]])
; CHECK-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP25]], <4 x float> [[TMP26]])
; CHECK-NEXT: [[TMP29:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX9]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP34]], true
+; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -283,7 +271,7 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x float> [[TMP20]], float [[TMP24]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
; CHECK: [[PRED_LOAD_CONTINUE7]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x float> [ [[TMP20]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[TMP56:%.*]] = phi <4 x float> [ [[TMP20]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], %[[PRED_LOAD_IF6]] ]
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
; CHECK: [[PRED_LOAD_IF8]]:
@@ -323,38 +311,32 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP49:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP48]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
; CHECK: [[PRED_LOAD_CONTINUE15]]:
-; CHECK-NEXT: [[TMP50:%.*]] = phi <4 x float> [ [[TMP44]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP49]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP65:%.*]] = phi <4 x float> [ [[TMP44]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP49]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x float> [[TMP56]]
+; CHECK-NEXT: [[TMP50:%.*]] = freeze <4 x float> [[TMP65]]
; CHECK-NEXT: [[TMP51]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP26]])
; CHECK-NEXT: [[TMP52]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP50]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP55:%.*]] = fcmp uno <4 x float> [[TMP26]], [[TMP50]]
-; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[TMP56]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP62:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT17]]
+; CHECK-NEXT: [[TMP57:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
; CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP57:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
; CHECK-NEXT: [[TMP59:%.*]] = or i1 [[TMP57]], [[TMP58]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: br i1 [[TMP59]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP53:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP51]], <4 x float> [[VEC_PHI]]
; CHECK-NEXT: [[TMP54:%.*]] = select <4 x i1> [[TMP2]], <4 x float> [[TMP52]], <4 x float> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP67:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
-; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP67]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP53]]
-; CHECK-NEXT: [[TMP69:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
-; CHECK-NEXT: [[TMP61:%.*]] = select i1 [[TMP69]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP54]]
-; CHECK-NEXT: [[TMP65:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
-; CHECK-NEXT: [[TMP66:%.*]] = select i1 [[TMP65]], i64 [[INDEX]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP53]]
+; CHECK-NEXT: [[TMP61:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP54]]
+; CHECK-NEXT: [[TMP62:%.*]] = select i1 [[TMP57]], i64 [[INDEX]], i64 [[TMP0]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP60]], <4 x float> [[TMP61]])
; CHECK-NEXT: [[TMP63:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX]])
-; CHECK-NEXT: [[TMP68:%.*]] = extractelement <4 x i1> [[TMP62]], i64 0
-; CHECK-NEXT: [[TMP64:%.*]] = xor i1 [[TMP68]], true
+; CHECK-NEXT: [[TMP64:%.*]] = xor i1 [[TMP57]], true
; CHECK-NEXT: br i1 [[TMP64]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP66]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP62]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[TMP63]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 8c5719996f6f02..6cca79fd5ea945 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -203,30 +203,24 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP2]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -274,30 +268,24 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -347,27 +335,21 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
; CHECK-NEXT: [[IV:%.*]] = add nuw i64 10, [[INDEX]]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP5]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP12]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP18]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP6]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = add i64 10, [[TMP9]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -421,27 +403,21 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
; CHECK-NEXT: [[IV:%.*]] = add nuw i64 [[START]], [[INDEX]]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP5]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP12]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP16]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP18]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP6]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[START]], [[TMP9]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP6]], true
; CHECK-NEXT: [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -718,34 +694,28 @@ define float @test_fmax_and_fmax(ptr %src.0, ptr %src.1, i64 %n) {
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC_0]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
-; CHECK-NEXT: [[TMP2]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[TMP4]])
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x float> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP20]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP2]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP9]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP16:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP13]])
; CHECK-NEXT: [[TMP17:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP14]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP22]], true
+; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP9]], true
; CHECK-NEXT: [[TMP19:%.*]] = and i1 [[CMP_N]], [[TMP18]]
; CHECK-NEXT: br i1 [[TMP19]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi float [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -844,33 +814,27 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 {
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP21]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x float> [ [[TMP17]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x float> [ [[TMP17]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x float> [[TMP26]]
; CHECK-NEXT: [[TMP24]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP23]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]]
-; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP36:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP0]], <4 x float> [[TMP24]], <4 x float> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
-; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP32]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]]
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
-; CHECK-NEXT: [[TMP35:%.*]] = select i1 [[TMP38]], i64 [[INDEX]], i64 30
+; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP27]], i64 [[INDEX]], i64 30
; CHECK-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP31]])
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP36]], i64 0
-; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP37]], true
+; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP27]], true
; CHECK-NEXT: br i1 [[TMP34]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP35]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP32]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[TMP33]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index ae5228d9431fba..a6a02fdc736c1b 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -203,30 +203,24 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP2]], <4 x float> [[VEC_PHI]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -274,30 +268,24 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP10]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp uno <4 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP3]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index bc6ce82606cf2f..eb490e74d9f41b 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -771,8 +771,11 @@ for.body:
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK: <2 x float> @llvm.minnum.v2f32
-; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
+; CHECK: [[FREEZE0:%.*]] = freeze <2 x float> [[OP0:%.+]]
+; CHECK-NEXT: [[FREEZE1:%.*]] = freeze <2 x float> [[OP1:%.+]]
+; CHECK-NEXT: <2 x float> @llvm.minnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE0]])
+; CHECK-NEXT: <2 x float> @llvm.minnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE1]])
+; CHECK: fcmp uno <2 x float> [[FREEZE0]], [[FREEZE1]]
define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
@@ -793,8 +796,11 @@ for.body:
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK: <2 x float> @llvm.maxnum.v2f32
-; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
+; CHECK: [[FREEZE0:%.*]] = freeze <2 x float> [[OP0:%.+]]
+; CHECK-NEXT: [[FREEZE1:%.*]] = freeze <2 x float> [[OP1:%.+]]
+; CHECK-NEXT: <2 x float> @llvm.maxnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE0]])
+; CHECK-NEXT: <2 x float> @llvm.maxnum.v2f32(<2 x float> {{%.+}}, <2 x float> [[FREEZE1]])
+; CHECK: fcmp uno <2 x float> [[FREEZE0]], [[FREEZE1]]
define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
index 68516d230711e2..64d5820f321299 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -1,9 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
-; Make sure we don't carry out the
-; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
-; combine, since the freezes should block it.
+; FIXME; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
+; replaces two individual freezes with one joint freeze which is incorrect.
+;
+; e.g. when x[0] = NaN and x[4] = poison the scalar loop is well defined and
+; returns true. When vectorized we get reduce.or(freeze(fcmp uno x[0]=NaN, x[4]=poison))
+; which may return false.
define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
; CHECK-LABEL: define i1 @find_nan(
>From a10ece428de63e587080f364019b2242fe0924e9 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 21 Sep 2026 09:35:02 +0100
Subject: [PATCH 9/9] Address comments
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 ++++++----
.../LoopVectorize/AArch64/cmp_cost.ll | 24 +++++++++----------
.../conditional-scalar-assignment-vplan.ll | 2 +-
.../single-early-exit-anyof-fold.ll | 8 +++----
4 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6ded858805e77f..83c4ecddfb6224 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3070,7 +3070,8 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
// EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
// EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
// EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
- // EMIT vp<%4> = any-of ir<%3>
+ // EMIT vp<%freeze> = freeze ir<%3>
+ // EMIT vp<%4> = any-of ir<%freeze>
// EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
// EMIT branch-on-two-conds vp<%4>, vp<%5>
// Successor(s): middle.block, middle.block, for.body
@@ -3123,11 +3124,12 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
return nullptr;
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
Recipes.push_back(cast<VPInstruction>(GepR));
- } else if (match(V, m_CombineOr(m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op1)),
- m_Freeze(m_VPValue(Op1))))) {
- Worklist.push_back(Op1);
+ } else if (match(V, m_Freeze(m_VPValue(
+ Op1, m_VPInstruction<VPInstruction::MaskedCond>(
+ m_VPValue(Op2)))))) {
+ Worklist.push_back(Op2);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
+ Recipes.push_back(cast<VPInstruction>(Op1->getDefiningRecipe()));
} else
return nullptr;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
index 83ec5cb9c9b574..7508461d4e4ec7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
@@ -185,8 +185,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP34:%[0-9]+]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP35:%[0-9]+]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP36:%[0-9]+]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4>
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 7 for VF 2: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP37:%[0-9]+]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -243,8 +243,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4>
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 13 for VF 4: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -301,8 +301,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
-; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 10 for VF 8: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 8: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -359,8 +359,8 @@ define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4> (!vplan.execution.frequency 542551298306736128 (5.882%, estimated))
-; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1> (!vplan.execution.frequency 8138269444536270848 (88.24%, estimated))
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 20 for VF 16: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 16: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
@@ -442,7 +442,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 1 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -465,7 +465,7 @@ define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 1 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -518,7 +518,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 2: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 8796093022208 (9.537e-05%, estimated))
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 2 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
@@ -541,7 +541,7 @@ define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 4: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
-; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00> (!vplan.execution.frequency 8796093022208 (9.537e-05%, estimated))
+; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 2 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index 5a9903b4e6e4e8..2c304bcef02fb2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -192,7 +192,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%if.cond> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
+; CHECK-NEXT: BRANCH-ON-MASK ir<%if.cond>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
index 64d5820f321299..53971f07d617c7 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-anyof-fold.ll
@@ -1,12 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
-; FIXME; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
-; replaces two individual freezes with one joint freeze which is incorrect.
+; Previously this applied the
+; any-of (fcmp uno A, A), (fcmp uno B, B), ...-> any-of (fcmp uno A, B)
+; combine and removed the freezes, which was incorrect.
;
-; e.g. when x[0] = NaN and x[4] = poison the scalar loop is well defined and
-; returns true. When vectorized we get reduce.or(freeze(fcmp uno x[0]=NaN, x[4]=poison))
-; which may return false.
define i1 @find_nan(ptr dereferenceable(32) align(16) %a) {
; CHECK-LABEL: define i1 @find_nan(
More information about the llvm-commits
mailing list