[llvm] [LV] Support interleaving with conditional scalar assignments (PR #184099)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 4 06:42:58 PST 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/184099
>From 49e2437315e9691ad6a7774cec4e1e941045cf6d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 2 Mar 2026 10:41:43 +0000
Subject: [PATCH 1/2] [LV] Support interleaving with conditional scalar
assignments
This extends the existing support to work with arbitrary interleave
factors. The main change here is reworking the ExtractLastActive
VPInstruction to take a variable amount of arguments and handling it
in unrollRecipeByUF and VPInstruction::generate.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 -
llvm/lib/Transforms/Vectorize/VPlan.h | 13 +-
.../Vectorize/VPlanConstruction.cpp | 2 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 7 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 25 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 15 +
.../AArch64/conditional-scalar-assignment.ll | 45 ++-
.../conditional-scalar-assignment-vplan.ll | 8 +-
...ave-conditional-scalar-assignment-vplan.ll | 185 ++++++++++
...ional-scalar-assignment-interleave-only.ll | 78 ++++
.../iv-select-cmp-non-const-iv-start.ll | 47 ++-
.../LoopVectorize/iv-select-cmp-trunc.ll | 337 +++++++++++++++---
.../Transforms/LoopVectorize/select-cmp.ll | 48 ++-
13 files changed, 719 insertions(+), 100 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 91c7f1680aac2..059763e8789e9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9767,15 +9767,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
- // FIXME: Enable interleaving for FindLast reductions.
- if (InterleaveLoop && hasFindLastReductionPhi(LVP.getPlanFor(VF.Width))) {
- LLVM_DEBUG(dbgs() << "LV: Not interleaving due to FindLast reduction.\n");
- IntDiagMsg = {"FindLastPreventsScalarInterleaving",
- "Unable to interleave due to FindLast reduction."};
- InterleaveLoop = false;
- IC = 1;
- }
-
// Emit diagnostic messages, if any.
const char *VAPassName = Hints.vectorizeAnalysisPassName();
if (!VectorizeLoop && !InterleaveLoop) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 41eef2a368343..b360098fc7770 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1294,9 +1294,11 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Explicit user for the resume phi of the canonical induction in the main
/// VPlan, used by the epilogue vector loop.
ResumeForEpilogue,
- /// Extracts the lane from the first operand corresponding to the last
- /// active (non-zero) lane in the mask (second operand), or if no lanes
- /// were active in the mask, returns the default value (third operand).
+ /// Extracts the last active lane from a set of vectors. The first operand
+ /// is the default value if no lanes are active. Conceptually, this
+ /// concatenates all data vectors (odd operands), concatenates all masks
+ /// (even operands -- ignoring the default value), and returns the last
+ /// active value from the combined data vector using the combined mask.
ExtractLastActive,
/// Returns the value for vscale.
@@ -2561,8 +2563,9 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
}
VPWidenPHIRecipe *clone() override {
- auto *C = new VPWidenPHIRecipe(cast<PHINode>(getUnderlyingValue()),
- getOperand(0), getDebugLoc(), Name);
+ auto *C =
+ new VPWidenPHIRecipe(cast_if_present<PHINode>(getUnderlyingValue()),
+ getOperand(0), getDebugLoc(), Name);
for (VPValue *Op : llvm::drop_begin(operands()))
C->addOperand(Op);
return C;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 83907fb96dbd2..b06a2612e3b05 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1425,7 +1425,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
Builder.setInsertPoint(RdxResult);
auto *ExtractLastActive =
Builder.createNaryOp(VPInstruction::ExtractLastActive,
- {DataSelect, MaskSelect, PhiR->getStartValue()},
+ {PhiR->getStartValue(), DataSelect, MaskSelect},
RdxResult->getDebugLoc());
RdxResult->replaceAllUsesWith(ExtractLastActive);
RdxResult->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 2b582a607dddc..1205f04fb5c29 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -508,6 +508,13 @@ m_LastActiveLane(const Op0_t &Op0) {
return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
}
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
+ Op2_t>
+m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+ return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
+}
+
template <typename Op0_t>
inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
m_ComputeReductionResult(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 26183f15306f1..0a21fb9628cea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -482,7 +482,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
case VPInstruction::ReductionStartVector:
- case VPInstruction::ExtractLastActive:
return 3;
case Instruction::Call: {
// For unmasked calls, the last argument will the called function. Use that
@@ -506,6 +505,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::SLPLoad:
case VPInstruction::SLPStore:
case VPInstruction::ExtractLane:
+ case VPInstruction::ExtractLastActive:
// Cannot determine the number of operands from the opcode.
return -1u;
}
@@ -905,13 +905,22 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::Reverse:
return Builder.CreateVectorReverse(State.get(getOperand(0)), "reverse");
case VPInstruction::ExtractLastActive: {
- Value *Data = State.get(getOperand(0));
- Value *Mask = State.get(getOperand(1));
- Value *Default = State.get(getOperand(2), /*IsScalar=*/true);
- Type *VTy = Data->getType();
- return Builder.CreateIntrinsic(
- Intrinsic::experimental_vector_extract_last_active, {VTy},
- {Data, Mask, Default});
+ Value *Default = State.get(getOperand(0), /*IsScalar=*/true);
+ Value *Result = Default;
+ for (unsigned Idx = 1; Idx < getNumOperands(); Idx += 2) {
+ Value *Data = State.get(getOperand(Idx));
+ Value *Mask = State.get(getOperand(Idx + 1));
+ Type *VTy = Data->getType();
+
+ if (State.VF.isScalar())
+ Result = Builder.CreateSelect(Mask, Data, Result);
+ else
+ Result = Builder.CreateIntrinsic(
+ Intrinsic::experimental_vector_extract_last_active, {VTy},
+ {Data, Mask, Result});
+ }
+
+ return Result;
}
default:
llvm_unreachable("Unsupported opcode for instruction");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 222d5d6d92b8c..5fbb86221c765 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -326,6 +326,10 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->insertBefore(VPBB, InsertPt);
addRecipeForPart(&R, Copy, Part);
+ // Phi operands are updated once all other recipes have been unrolled.
+ if (isa<VPWidenPHIRecipe>(Copy))
+ continue;
+
VPValue *Op;
if (match(&R, m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
m_VPValue(), m_VPValue(Op)))) {
@@ -437,6 +441,17 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
+ VPValue *Op2;
+ if (match(&R, m_ExtractLastActive(m_VPValue(), m_VPValue(Op1),
+ m_VPValue(Op2)))) {
+ addUniformForAllParts(cast<VPInstruction>(&R));
+ for (unsigned Part = 1; Part != UF; ++Part) {
+ R.addOperand(getValueForPart(Op1, Part));
+ R.addOperand(getValueForPart(Op2, Part));
+ }
+ continue;
+ }
+
if (Plan.hasScalarVFOnly()) {
if (match(&R, m_ExtractLastPart(m_VPValue(Op0))) ||
match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
index 7053aa60b2035..df2d6c8c775a4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
@@ -710,10 +710,10 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON-LABEL: define i32 @simple_csa_int_select_use_interleave(
; NEON-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) {
; NEON-NEXT: [[ENTRY:.*]]:
-; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NEON: [[VECTOR_PH]]:
-; NEON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; NEON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; NEON-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; NEON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
; NEON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -721,24 +721,34 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON: [[VECTOR_BODY]]:
; NEON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; NEON-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; NEON-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
+; NEON-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[LD_ADDR]], i64 4
; NEON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[LD_ADDR]], align 4
+; NEON-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
; NEON-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; NEON-NEXT: [[TMP10:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD2]]
; NEON-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; NEON-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; NEON-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP10]]
+; NEON-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP3]], [[TMP12]]
+; NEON-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
; NEON-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; NEON-NEXT: [[TMP11]] = select i1 [[TMP4]], <4 x i1> [[TMP10]], <4 x i1> [[TMP1]]
; NEON-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
-; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; NEON-NEXT: [[TMP13]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[VEC_PHI1]]
+; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; NEON-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NEON-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; NEON: [[MIDDLE_BLOCK]]:
; NEON-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP6]], <4 x i1> [[TMP5]], i32 -1)
+; NEON-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP13]], <4 x i1> [[TMP11]], i32 [[TMP8]])
; NEON-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; NEON-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; NEON: [[SCALAR_PH]]:
; NEON-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NEON-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; NEON-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; NEON-NEXT: br label %[[LOOP:.*]]
; NEON: [[LOOP]]:
; NEON-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -751,20 +761,21 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; NEON-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; NEON: [[EXIT]]:
-; NEON-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; NEON-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; NEON-NEXT: ret i32 [[SELECT_DATA_LCSSA]]
;
; SVE-LABEL: define i32 @simple_csa_int_select_use_interleave(
; SVE-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
; SVE-NEXT: [[ENTRY:.*]]:
; SVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; SVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; SVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; SVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; SVE: [[VECTOR_PH]]:
; SVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; SVE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; SVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; SVE-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 1
+; SVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP14]]
; SVE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; SVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A]], i64 0
; SVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -772,24 +783,34 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; SVE: [[VECTOR_BODY]]:
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP15:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
+; SVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP3]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; SVE-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
; SVE-NEXT: [[TMP6:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; SVE-NEXT: [[TMP19:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD2]]
; SVE-NEXT: [[TMP7:%.*]] = freeze <vscale x 4 x i1> [[TMP6]]
-; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP7]])
+; SVE-NEXT: [[TMP20:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
+; SVE-NEXT: [[TMP13:%.*]] = or <vscale x 4 x i1> [[TMP7]], [[TMP20]]
+; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP13]])
; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP16]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i1> [[TMP15]]
; SVE-NEXT: [[TMP10]] = select i1 [[TMP8]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; SVE-NEXT: [[TMP18]] = select i1 [[TMP8]], <vscale x 4 x i32> [[WIDE_LOAD2]], <vscale x 4 x i32> [[VEC_PHI1]]
+; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
; SVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; SVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; SVE: [[MIDDLE_BLOCK]]:
; SVE-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i1> [[TMP9]], i32 -1)
+; SVE-NEXT: [[TMP21:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x i1> [[TMP16]], i32 [[TMP12]])
; SVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; SVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; SVE: [[SCALAR_PH]]:
; SVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; SVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; SVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; SVE-NEXT: br label %[[LOOP:.*]]
; SVE: [[LOOP]]:
; SVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -802,7 +823,7 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; SVE-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; SVE-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; SVE: [[EXIT]]:
-; SVE-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; SVE-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
; SVE-NEXT: ret i32 [[SELECT_DATA_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index cb18a22409641..81855a314945d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -49,7 +49,7 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-active vp<[[VP9]]>, vp<[[VP8]]>, ir<-1>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP9]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -112,7 +112,7 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-last-active vp<[[VP13]]>, vp<[[VP12]]>, ir<-1>
+; CHECK-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
@@ -216,7 +216,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-active vp<[[VP11]]>, vp<[[VP10]]>, ir<%default_val>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP11]]>, vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -280,7 +280,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active vp<[[VP15]]>, vp<[[VP14]]>, ir<%default_val>
+; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
new file mode 100644
index 0000000000000..8a3bab199abc5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -vplan-print-after="printFinalVPlan$" \
+; RUN: -force-vector-width=4 -force-vector-interleave=4 -force-target-supports-masked-memory-ops -disable-output 2>&1 < %s | FileCheck %s --check-prefix=IC4
+; RUN: opt -passes=loop-vectorize -vplan-print-after="printFinalVPlan$" \
+; RUN: -force-vector-width=4 -force-vector-interleave=4 -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize \
+; RUN: -disable-output 2>&1 < %s | FileCheck %s --check-prefix=IC4-TF
+
+; This function is derived from the following C program:
+; int find_last_int_select(int N, int *data, int a) {
+; int t = -1;
+; for (int i = 0; i < N; i++) {
+; if (a < data[i])
+; t = data[i];
+; }
+; return t;
+; }
+define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
+; IC4-LABEL: VPlan for loop in 'find_last_int_select'
+; IC4: VPlan 'Final VPlan for VF={4},UF={4}' {
+; IC4-NEXT: Live-in ir<%N> = original trip-count
+; IC4-EMPTY:
+; IC4-NEXT: ir-bb<entry>:
+; IC4-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%N>, ir<16>
+; IC4-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; IC4-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+; IC4-EMPTY:
+; IC4-NEXT: vector.ph:
+; IC4-NEXT: EMIT vp<%n.mod.vf> = urem ir<%N>, ir<16>
+; IC4-NEXT: EMIT vp<%n.vec> = sub ir<%N>, vp<%n.mod.vf>
+; IC4-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = broadcast ir<%a>
+; IC4-NEXT: Successor(s): vector.body
+; IC4-EMPTY:
+; IC4-NEXT: vector.body:
+; IC4-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; IC4-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP16:%[0-9]+]]>
+; IC4-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi ir<-1>, vp<[[VP17:%[0-9]+]]>
+; IC4-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.2 = phi ir<-1>, vp<[[VP18:%[0-9]+]]>
+; IC4-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.3 = phi ir<-1>, vp<[[VP19:%[0-9]+]]>
+; IC4-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP12:%[0-9]+]]>, vector.body ]
+; IC4-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
+; IC4-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
+; IC4-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP15:%[0-9]+]]>, vector.body ]
+; IC4-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
+; IC4-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<4>
+; IC4-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<8>
+; IC4-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<12>
+; IC4-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>
+; IC4-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP8]]>
+; IC4-NEXT: WIDEN ir<%ld>.2 = load vp<[[VP9]]>
+; IC4-NEXT: WIDEN ir<%ld>.3 = load vp<[[VP10]]>
+; IC4-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP3]]>, ir<%ld>
+; IC4-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP3]]>, ir<%ld>.1
+; IC4-NEXT: WIDEN ir<%select.cmp>.2 = icmp slt vp<[[VP3]]>, ir<%ld>.2
+; IC4-NEXT: WIDEN ir<%select.cmp>.3 = icmp slt vp<[[VP3]]>, ir<%ld>.3
+; IC4-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of ir<%select.cmp>, ir<%select.cmp>.1, ir<%select.cmp>.2, ir<%select.cmp>.3
+; IC4-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP11]]>, ir<%select.cmp>, vp<[[VP4]]>
+; IC4-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP11]]>, ir<%select.cmp>.1, vp<[[VP5]]>
+; IC4-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP11]]>, ir<%select.cmp>.2, vp<[[VP6]]>
+; IC4-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP11]]>, ir<%select.cmp>.3, vp<[[VP7]]>
+; IC4-NEXT: EMIT vp<[[VP16]]> = select vp<[[VP11]]>, ir<%ld>, ir<%data.phi>
+; IC4-NEXT: EMIT vp<[[VP17]]> = select vp<[[VP11]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC4-NEXT: EMIT vp<[[VP18]]> = select vp<[[VP11]]>, ir<%ld>.2, ir<%data.phi>.2
+; IC4-NEXT: EMIT vp<[[VP19]]> = select vp<[[VP11]]>, ir<%ld>.3, ir<%data.phi>.3
+; IC4-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<16>
+; IC4-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC4-NEXT: EMIT branch-on-cond vp<[[VP20]]>
+; IC4-NEXT: Successor(s): middle.block, vector.body
+; IC4-EMPTY:
+; IC4-NEXT: middle.block:
+; IC4-NEXT: EMIT vp<[[VP22:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP16]]>, vp<[[VP12]]>, vp<[[VP17]]>, vp<[[VP13]]>, vp<[[VP18]]>, vp<[[VP14]]>, vp<[[VP19]]>, vp<[[VP15]]>
+; IC4-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%N>, vp<%n.vec>
+; IC4-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; IC4-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
+; IC4-EMPTY:
+; IC4-NEXT: ir-bb<exit>:
+; IC4-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP22]]> from middle.block)
+; IC4-NEXT: No successors
+; IC4-EMPTY:
+; IC4-NEXT: ir-bb<scalar.ph>:
+; IC4-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%n.vec>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; IC4-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP22]]>, middle.block ], [ ir<-1>, ir-bb<entry> ]
+; IC4-NEXT: Successor(s): ir-bb<loop>
+; IC4-EMPTY:
+; IC4-NEXT: ir-bb<loop>:
+; IC4-NEXT: IR %iv = phi i64 [ 0, %scalar.ph ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from ir-bb<scalar.ph>)
+; IC4-NEXT: IR %data.phi = phi i32 [ -1, %scalar.ph ], [ %select.data, %loop ] (extra operand: vp<%bc.merge.rdx> from ir-bb<scalar.ph>)
+; IC4-NEXT: IR %ld.addr = getelementptr inbounds i32, ptr %data, i64 %iv
+; IC4-NEXT: IR %ld = load i32, ptr %ld.addr, align 4
+; IC4-NEXT: IR %select.cmp = icmp slt i32 %a, %ld
+; IC4-NEXT: IR %select.data = select i1 %select.cmp, i32 %ld, i32 %data.phi
+; IC4-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
+; IC4-NEXT: IR %exit.cmp = icmp eq i64 %iv.next, %N
+; IC4-NEXT: No successors
+; IC4-NEXT: }
+;
+; IC4-TF-LABEL: VPlan for loop in 'find_last_int_select'
+; IC4-TF: VPlan 'Final VPlan for VF={4},UF={4}' {
+; IC4-TF-NEXT: Live-in ir<%N> = original trip-count
+; IC4-TF-EMPTY:
+; IC4-TF-NEXT: ir-bb<entry>:
+; IC4-TF-NEXT: Successor(s): vector.ph
+; IC4-TF-EMPTY:
+; IC4-TF-NEXT: vector.ph:
+; IC4-TF-NEXT: EMIT vp<%n.rnd.up> = add ir<%N>, ir<15>
+; IC4-TF-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, ir<16>
+; IC4-TF-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<%n.mod.vf>
+; IC4-TF-NEXT: EMIT vp<%trip.count.minus.1> = sub ir<%N>, ir<1>
+; IC4-TF-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; IC4-TF-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = broadcast ir<%a>
+; IC4-TF-NEXT: Successor(s): vector.body
+; IC4-TF-EMPTY:
+; IC4-TF-NEXT: vector.body:
+; IC4-TF-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; IC4-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP28:%[0-9]+]]>
+; IC4-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi ir<-1>, vp<[[VP29:%[0-9]+]]>
+; IC4-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.2 = phi ir<-1>, vp<[[VP30:%[0-9]+]]>
+; IC4-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.3 = phi ir<-1>, vp<[[VP31:%[0-9]+]]>
+; IC4-TF-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP24:%[0-9]+]]>, vector.body ]
+; IC4-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP25:%[0-9]+]]>, vector.body ]
+; IC4-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP26:%[0-9]+]]>, vector.body ]
+; IC4-TF-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP27:%[0-9]+]]>, vector.body ]
+; IC4-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
+; IC4-TF-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>, ir<1>
+; IC4-TF-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>, ir<2>
+; IC4-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>, ir<3>
+; IC4-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP2]]>
+; IC4-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp ule vp<[[VP9]]>, vp<[[VP2]]>
+; IC4-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp ule vp<[[VP10]]>, vp<[[VP2]]>
+; IC4-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp ule vp<[[VP11]]>, vp<[[VP2]]>
+; IC4-TF-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
+; IC4-TF-NEXT: vp<[[VP16:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<4>
+; IC4-TF-NEXT: vp<[[VP17:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<8>
+; IC4-TF-NEXT: vp<[[VP18:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<12>
+; IC4-TF-NEXT: WIDEN ir<%ld> = load ir<%ld.addr>, vp<[[VP12]]>
+; IC4-TF-NEXT: WIDEN ir<%ld>.1 = load vp<[[VP16]]>, vp<[[VP13]]>
+; IC4-TF-NEXT: WIDEN ir<%ld>.2 = load vp<[[VP17]]>, vp<[[VP14]]>
+; IC4-TF-NEXT: WIDEN ir<%ld>.3 = load vp<[[VP18]]>, vp<[[VP15]]>
+; IC4-TF-NEXT: WIDEN ir<%select.cmp> = icmp slt vp<[[VP3]]>, ir<%ld>
+; IC4-TF-NEXT: WIDEN ir<%select.cmp>.1 = icmp slt vp<[[VP3]]>, ir<%ld>.1
+; IC4-TF-NEXT: WIDEN ir<%select.cmp>.2 = icmp slt vp<[[VP3]]>, ir<%ld>.2
+; IC4-TF-NEXT: WIDEN ir<%select.cmp>.3 = icmp slt vp<[[VP3]]>, ir<%ld>.3
+; IC4-TF-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP12]]>, ir<%select.cmp>
+; IC4-TF-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP13]]>, ir<%select.cmp>.1
+; IC4-TF-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and vp<[[VP14]]>, ir<%select.cmp>.2
+; IC4-TF-NEXT: EMIT vp<[[VP22:%[0-9]+]]> = logical-and vp<[[VP15]]>, ir<%select.cmp>.3
+; IC4-TF-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = any-of vp<[[VP19]]>, vp<[[VP20]]>, vp<[[VP21]]>, vp<[[VP22]]>
+; IC4-TF-NEXT: EMIT vp<[[VP24]]> = select vp<[[VP23]]>, vp<[[VP19]]>, vp<[[VP4]]>
+; IC4-TF-NEXT: EMIT vp<[[VP25]]> = select vp<[[VP23]]>, vp<[[VP20]]>, vp<[[VP5]]>
+; IC4-TF-NEXT: EMIT vp<[[VP26]]> = select vp<[[VP23]]>, vp<[[VP21]]>, vp<[[VP6]]>
+; IC4-TF-NEXT: EMIT vp<[[VP27]]> = select vp<[[VP23]]>, vp<[[VP22]]>, vp<[[VP7]]>
+; IC4-TF-NEXT: EMIT vp<[[VP28]]> = select vp<[[VP23]]>, ir<%ld>, ir<%data.phi>
+; IC4-TF-NEXT: EMIT vp<[[VP29]]> = select vp<[[VP23]]>, ir<%ld>.1, ir<%data.phi>.1
+; IC4-TF-NEXT: EMIT vp<[[VP30]]> = select vp<[[VP23]]>, ir<%ld>.2, ir<%data.phi>.2
+; IC4-TF-NEXT: EMIT vp<[[VP31]]> = select vp<[[VP23]]>, ir<%ld>.3, ir<%data.phi>.3
+; IC4-TF-NEXT: EMIT vp<%index.next> = add vp<%index>, ir<16>
+; IC4-TF-NEXT: EMIT vp<[[VP32:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; IC4-TF-NEXT: EMIT branch-on-cond vp<[[VP32]]>
+; IC4-TF-NEXT: Successor(s): middle.block, vector.body
+; IC4-TF-EMPTY:
+; IC4-TF-NEXT: middle.block:
+; IC4-TF-NEXT: EMIT vp<[[VP34:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP28]]>, vp<[[VP24]]>, vp<[[VP29]]>, vp<[[VP25]]>, vp<[[VP30]]>, vp<[[VP26]]>, vp<[[VP31]]>, vp<[[VP27]]>
+; IC4-TF-NEXT: Successor(s): ir-bb<exit>
+; IC4-TF-EMPTY:
+; IC4-TF-NEXT: ir-bb<exit>:
+; IC4-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP34]]> from middle.block)
+; IC4-TF-NEXT: No successors
+; IC4-TF-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %data.phi = phi i32 [ -1, %entry ], [ %select.data, %loop ]
+ %ld.addr = getelementptr inbounds i32, ptr %data, i64 %iv
+ %ld = load i32, ptr %ld.addr, align 4
+ %select.cmp = icmp slt i32 %a, %ld
+ %select.data = select i1 %select.cmp, i32 %ld, i32 %data.phi
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit.cmp = icmp eq i64 %iv.next, %N
+ br i1 %exit.cmp, label %exit, label %loop
+
+exit:
+ ret i32 %select.data
+}
diff --git a/llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll
new file mode 100644
index 0000000000000..92e6b36edf17e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -S < %s 2>&1 | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+define i32 @simple_find_last_reduction(i64 %N, ptr %data, i32 %a) {
+; CHECK-LABEL: define i32 @simple_find_last_reduction(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[A]], [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp slt i32 [[A]], [[TMP21]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12]] = select i1 [[TMP11]], i1 [[TMP7]], i1 [[TMP0]]
+; CHECK-NEXT: [[TMP13]] = select i1 [[TMP11]], i1 [[TMP8]], i1 [[TMP3]]
+; CHECK-NEXT: [[TMP14]] = select i1 [[TMP11]], i32 [[TMP5]], i32 [[VEC_PHI]]
+; CHECK-NEXT: [[TMP19]] = select i1 [[TMP11]], i32 [[TMP21]], i32 [[VEC_PHI1]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP2]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP12]], i32 [[TMP14]], i32 -1
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP13]], i32 [[TMP19]], i32 [[TMP17]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[TMP2]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[DATA_PHI:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT_DATA:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[LD_ADDR]], align 4
+; CHECK-NEXT: [[SELECT_CMP:%.*]] = icmp slt i32 [[A]], [[LD]]
+; CHECK-NEXT: [[SELECT_DATA]] = select i1 [[SELECT_CMP]], i32 [[LD]], i32 [[DATA_PHI]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ [[SELECT_DATA]], %[[EXIT]] ], [ [[TMP18]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[TMP15]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %data.phi = phi i32 [ -1, %entry ], [ %select.data, %loop ]
+ %ld.addr = getelementptr inbounds i32, ptr %data, i64 %iv
+ %ld = load i32, ptr %ld.addr, align 4
+ %select.cmp = icmp slt i32 %a, %ld
+ %select.data = select i1 %select.cmp, i32 %ld, i32 %data.phi
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit.cmp = icmp eq i64 %iv.next, %N
+ br i1 %exit.cmp, label %exit, label %loop
+
+exit:
+ ret i32 %select.data
+}
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
index 0e07de9d5b403..0bc5987656c7a 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -332,10 +332,10 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = sext i32 [[IV_START]] to i64
; CHECK-VF4IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = sext i32 [[N]] to i64
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[TMP0]]
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[RDX_START]], i64 0
@@ -347,28 +347,61 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP23:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-VF4IC4-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP31]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP32]], align 4
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD7]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD8]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP33]]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = or <4 x i1> [[TMP5]], [[TMP16]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP37]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP17]], [[TMP18]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = or <4 x i1> [[TMP19]], [[TMP20]]
+; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]])
; CHECK-VF4IC4-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP24]] = select i1 [[TMP6]], <4 x i1> [[TMP33]], <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP6]], <4 x i1> [[TMP37]], <4 x i1> [[TMP22]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP6]], <4 x i1> [[TMP14]], <4 x i1> [[TMP23]]
; CHECK-VF4IC4-NEXT: [[TMP8]] = select i1 [[TMP6]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP28]] = select i1 [[TMP6]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[TMP29]] = select i1 [[TMP6]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI4]]
+; CHECK-VF4IC4-NEXT: [[TMP30]] = select i1 [[TMP6]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI5]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP8]], <4 x i1> [[TMP7]], i32 [[TMP10]])
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP28]], <4 x i1> [[TMP24]], i32 [[TMP11]])
+; CHECK-VF4IC4-NEXT: [[TMP35:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP29]], <4 x i1> [[TMP25]], i32 [[TMP34]])
+; CHECK-VF4IC4-NEXT: [[TMP36:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP30]], <4 x i1> [[TMP26]], i32 [[TMP35]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP36]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ]
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -382,7 +415,7 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-VF4IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP36]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: br label %[[EXIT]]
; CHECK-VF4IC4: [[EXIT]]:
; CHECK-VF4IC4-NEXT: [[IDX_0_LCSSA:%.*]] = phi i32 [ [[RDX_START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index ceb06f2e3971e..4381ac4ff3cfc 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -744,35 +744,68 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF4IC4-NEXT: [[CMP_SGT:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-VF4IC4-NEXT: br i1 [[CMP_SGT]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-VF4IC4: [[FOR_BODY_PREHEADER]]:
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP33]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP3]], <4 x i1> [[TMP33]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP3]], <4 x i1> [[TMP10]], <4 x i1> [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP3]], <4 x i1> [[TMP11]], <4 x i1> [[TMP20]]
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP5]], <4 x i1> [[TMP4]], i32 331)
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP25]], <4 x i1> [[TMP21]], i32 [[TMP7]])
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP26]], <4 x i1> [[TMP22]], i32 [[TMP30]])
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 [[TMP31]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP32]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -786,7 +819,7 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-VF4IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP32]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: br label %[[EXIT]]
; CHECK-VF4IC4: [[EXIT]]:
; CHECK-VF4IC4-NEXT: [[RDX_LCSSA:%.*]] = phi i32 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -910,35 +943,68 @@ define i32 @select_icmp_const_truncated_iv_unsigned_loop_guard(ptr %a, i32 %n) {
; CHECK-VF4IC4-NEXT: br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER:.*]]
; CHECK-VF4IC4: [[FOR_BODY_PREHEADER]]:
; CHECK-VF4IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP29]], align 4
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP33]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP3]], <4 x i1> [[TMP33]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP3]], <4 x i1> [[TMP10]], <4 x i1> [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP3]], <4 x i1> [[TMP11]], <4 x i1> [[TMP20]]
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP5]], <4 x i1> [[TMP4]], i32 331)
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP25]], <4 x i1> [[TMP21]], i32 [[TMP7]])
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP26]], <4 x i1> [[TMP22]], i32 [[TMP30]])
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 [[TMP31]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP32]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
@@ -952,7 +1018,7 @@ define i32 @select_icmp_const_truncated_iv_unsigned_loop_guard(ptr %a, i32 %n) {
; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-VF4IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP32]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: br label %[[EXIT]]
; CHECK-VF4IC4: [[EXIT]]:
; CHECK-VF4IC4-NEXT: [[RDX_LCSSA:%.*]] = phi i32 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1055,25 +1121,58 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 -2, i32 -1, i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = add i64 4294967294, [[INDEX]]
; CHECK-VF4IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP20]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP1]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP3]], <4 x i1> [[TMP9]], <4 x i1> [[TMP8]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP3]], <4 x i1> [[TMP10]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP3]], <4 x i1> [[TMP11]], <4 x i1> [[TMP19]]
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9223372032559808512
; CHECK-VF4IC4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP5]], <4 x i1> [[TMP4]], i32 331)
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP25]], <4 x i1> [[TMP21]], i32 [[TMP7]])
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP26]], <4 x i1> [[TMP22]], i32 [[TMP30]])
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 [[TMP31]])
; CHECK-VF4IC4-NEXT: br label %[[EXIT:.*]]
; CHECK-VF4IC4: [[EXIT]]:
-; CHECK-VF4IC4-NEXT: ret i32 [[TMP7]]
+; CHECK-VF4IC4-NEXT: ret i32 [[TMP32]]
;
; CHECK-VF1IC4-LABEL: define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(
; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
@@ -1176,10 +1275,10 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF4IC4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[START:%.*]], i32 [[N:%.*]]) {
; CHECK-VF4IC4-NEXT: [[ENTRY:.*]]:
; CHECK-VF4IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[START]], i64 0
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -1187,29 +1286,68 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP23:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP32]], align 4
; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP33]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP38]], align 4
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
-; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
+; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP4]], <4 x i1> [[TMP13]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP4]], <4 x i1> [[TMP14]], <4 x i1> [[TMP16]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP4]], <4 x i1> [[TMP15]], <4 x i1> [[TMP23]]
; CHECK-VF4IC4-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP29]] = select i1 [[TMP4]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP30]] = select i1 [[TMP4]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP31]] = select i1 [[TMP4]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP6]], <4 x i1> [[TMP5]], i32 [[TMP8]])
+; CHECK-VF4IC4-NEXT: [[TMP35:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP29]], <4 x i1> [[TMP25]], i32 [[TMP9]])
+; CHECK-VF4IC4-NEXT: [[TMP36:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP30]], <4 x i1> [[TMP26]], i32 [[TMP35]])
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP31]], <4 x i1> [[TMP27]], i32 [[TMP36]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
@@ -1225,7 +1363,7 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-VF4IC4: [[EXIT]]:
-; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP37]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: ret i32 [[COND_LCSSA]]
;
; CHECK-VF1IC4-LABEL: define i32 @not_vectorized_select_iv_icmp_no_guard(
@@ -1328,27 +1466,60 @@ define i32 @not_vectorized_select_fcmp_invalid_const_ub(ptr %a) {
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i32> [[STEP_ADD_2]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds float, ptr [[ARRAYIDX]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[ARRAYIDX]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[ARRAYIDX]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP28]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP29]], align 4
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = fcmp fast olt <4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = fcmp fast olt <4 x float> [[WIDE_LOAD5]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = fcmp fast olt <4 x float> [[WIDE_LOAD6]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP8]]
-; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP2]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
; CHECK-VF4IC4-NEXT: [[TMP4]] = select i1 [[TMP3]], <4 x i1> [[TMP8]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP3]], <4 x i1> [[TMP9]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP3]], <4 x i1> [[TMP10]], <4 x i1> [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP3]], <4 x i1> [[TMP11]], <4 x i1> [[TMP20]]
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP3]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_2]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP3]], <4 x i32> [[STEP_ADD_3]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2147483648
; CHECK-VF4IC4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP5]], <4 x i1> [[TMP4]], i32 -1)
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP25]], <4 x i1> [[TMP21]], i32 [[TMP7]])
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP26]], <4 x i1> [[TMP22]], i32 [[TMP30]])
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 [[TMP31]])
; CHECK-VF4IC4-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV1:%.*]] = phi i64 [ 2147483648, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[TMP7]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[TMP32]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
; CHECK-VF4IC4-NEXT: [[CMP:%.*]] = fcmp fast olt float [[TMP0]], 0.000000e+00
@@ -1472,10 +1643,10 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF4IC4-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-VF4IC4: [[FOR_BODY_PREHEADER]]:
; CHECK-VF4IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[START]], i64 0
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
@@ -1483,29 +1654,68 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i16> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i16> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i16> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i16> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP23:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 4)
; CHECK-VF4IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP32]], align 4
; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP33]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x i32>, ptr [[TMP34]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP38]], align 4
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP11]]
-; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP3]], [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP19]]
+; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
+; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP4]], <4 x i1> [[TMP13]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP4]], <4 x i1> [[TMP14]], <4 x i1> [[TMP16]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP4]], <4 x i1> [[TMP15]], <4 x i1> [[TMP23]]
; CHECK-VF4IC4-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x i16> [[VEC_IND]], <4 x i16> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; CHECK-VF4IC4-NEXT: [[TMP29]] = select i1 [[TMP4]], <4 x i16> [[STEP_ADD]], <4 x i16> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP30]] = select i1 [[TMP4]], <4 x i16> [[STEP_ADD_2]], <4 x i16> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP31]] = select i1 [[TMP4]], <4 x i16> [[STEP_ADD_3]], <4 x i16> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD_3]], splat (i16 4)
; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[BROADCAST_SPLAT]], i32 0
; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = call i16 @llvm.experimental.vector.extract.last.active.v4i16(<4 x i16> [[TMP6]], <4 x i1> [[TMP5]], i16 [[TMP8]])
+; CHECK-VF4IC4-NEXT: [[TMP35:%.*]] = call i16 @llvm.experimental.vector.extract.last.active.v4i16(<4 x i16> [[TMP29]], <4 x i1> [[TMP25]], i16 [[TMP9]])
+; CHECK-VF4IC4-NEXT: [[TMP36:%.*]] = call i16 @llvm.experimental.vector.extract.last.active.v4i16(<4 x i16> [[TMP30]], <4 x i1> [[TMP26]], i16 [[TMP35]])
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = call i16 @llvm.experimental.vector.extract.last.active.v4i16(<4 x i16> [[TMP31]], <4 x i1> [[TMP27]], i16 [[TMP36]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF4IC4: [[FOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
@@ -1521,7 +1731,7 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK-VF4IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i16 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[COND_LCSSA:%.*]] = phi i16 [ [[COND]], %[[FOR_BODY]] ], [ [[TMP37]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: br label %[[EXIT]]
; CHECK-VF4IC4: [[EXIT]]:
; CHECK-VF4IC4-NEXT: [[RDX_0_LCSSA:%.*]] = phi i16 [ [[START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1662,10 +1872,10 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF4IC4-SAME: i64 [[N:%.*]], i64 [[START:%.*]]) {
; CHECK-VF4IC4-NEXT: [[ENTRY:.*]]:
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = add i64 [[START]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[START]], i64 0
@@ -1675,30 +1885,61 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF4IC4: [[LOOP]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC4-NEXT: [[VEC_IND1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[LOOP]] ]
; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
-; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ <i1 false, i1 true, i1 false, i1 true>, %[[VECTOR_PH]] ], [ [[VEC_PHI]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_IND1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_IND4:%.*]] = phi <4 x i1> [ <i1 false, i1 true, i1 false, i1 true>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD5:%.*]] = add <4 x i1> [[VEC_IND4]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_26:%.*]] = add <4 x i1> [[STEP_ADD5]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_37:%.*]] = add <4 x i1> [[STEP_ADD_26]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = icmp ne <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = icmp ne <4 x i64> [[STEP_ADD]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = icmp ne <4 x i64> [[STEP_ADD_2]], zeroinitializer
+; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[STEP_ADD_3]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
-; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[VEC_PHI]]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP18]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP12]], [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP14]], [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
; CHECK-VF4IC4-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> [[TMP2]]
; CHECK-VF4IC4-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i1> [[VEC_PHI]], <4 x i1> [[VEC_IND1]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[TMP20]] = select i1 [[TMP5]], <4 x i1> [[TMP18]], <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP5]], <4 x i1> [[TMP9]], <4 x i1> [[TMP17]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP5]], <4 x i1> [[VEC_IND4]], <4 x i1> [[VEC_PHI4]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP5]], <4 x i1> [[STEP_ADD5]], <4 x i1> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP24]] = select i1 [[TMP5]], <4 x i1> [[STEP_ADD_26]], <4 x i1> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP5]], <4 x i1> [[STEP_ADD_37]], <4 x i1> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT8]] = add <4 x i1> [[STEP_ADD_37]], zeroinitializer
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
-; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = call i1 @llvm.experimental.vector.extract.last.active.v4i1(<4 x i1> [[TMP7]], <4 x i1> [[TMP6]], i1 false)
+; CHECK-VF4IC4-NEXT: [[TMP27:%.*]] = call i1 @llvm.experimental.vector.extract.last.active.v4i1(<4 x i1> [[TMP22]], <4 x i1> [[TMP6]], i1 false)
+; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = call i1 @llvm.experimental.vector.extract.last.active.v4i1(<4 x i1> [[TMP23]], <4 x i1> [[TMP7]], i1 [[TMP27]])
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = call i1 @llvm.experimental.vector.extract.last.active.v4i1(<4 x i1> [[TMP24]], <4 x i1> [[TMP20]], i1 [[TMP28]])
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = call i1 @llvm.experimental.vector.extract.last.active.v4i1(<4 x i1> [[TMP25]], <4 x i1> [[TMP21]], i1 [[TMP29]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ false, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL9:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[TMP30]], %[[MIDDLE_BLOCK]] ], [ false, %[[ENTRY]] ]
; CHECK-VF4IC4-NEXT: br label %[[LOOP1:.*]]
; CHECK-VF4IC4: [[LOOP1]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-VF4IC4-NEXT: [[CTR:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[CTR_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-VF4IC4-NEXT: [[CTR:%.*]] = phi i64 [ [[BC_RESUME_VAL9]], %[[SCALAR_PH]] ], [ [[CTR_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-VF4IC4-NEXT: [[ACCUM:%.*]] = phi i1 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP1]] ]
; CHECK-VF4IC4-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 0
; CHECK-VF4IC4-NEXT: [[TRUNC:%.*]] = trunc i64 [[CTR]] to i1
@@ -1708,7 +1949,7 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF4IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[N]], [[CTR]]
; CHECK-VF4IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK-VF4IC4: [[EXIT]]:
-; CHECK-VF4IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], %[[LOOP1]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], %[[LOOP1]] ], [ [[TMP30]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: ret i1 [[SEL_LCSSA]]
;
; CHECK-VF1IC4-LABEL: define i1 @select_with_trunc_i1_iv(
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index 83d2e0281dfe1..a7f12136c4ae0 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -1181,35 +1181,71 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF4IC4-LABEL: define i32 @select_variant_i32_from_icmp(
; CHECK-VF4IC4-SAME: ptr [[V1:%.*]], ptr [[V2:%.*]], i64 [[N:%.*]]) {
; CHECK-VF4IC4-NEXT: [[ENTRY:.*]]:
-; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-VF4IC4: [[VECTOR_PH]]:
-; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF4IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-VF4IC4: [[VECTOR_BODY]]:
; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 3), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 3), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 3), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 3), %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[LAST_ACTIVE_MASK:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 12
; CHECK-VF4IC4-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD9]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP11]]
-; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
+; CHECK-VF4IC4-NEXT: [[TMP24:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP17]], [[TMP24]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = or <4 x i1> [[TMP18]], [[TMP33]]
+; CHECK-VF4IC4-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-VF4IC4-NEXT: [[TMP22:%.*]] = or <4 x i1> [[TMP20]], [[TMP21]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP22]])
; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> [[LAST_ACTIVE_MASK]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP19]], <4 x i1> [[TMP13]], <4 x i1> [[TMP1]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP19]], <4 x i1> [[TMP14]], <4 x i1> [[TMP2]]
+; CHECK-VF4IC4-NEXT: [[TMP32]] = select i1 [[TMP19]], <4 x i1> [[TMP15]], <4 x i1> [[TMP3]]
; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP19]], <4 x i32> [[WIDE_LOAD13]], <4 x i32> [[VEC_PHI]]
-; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC4-NEXT: [[TMP29]] = select i1 [[TMP19]], <4 x i32> [[WIDE_LOAD8]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP30]] = select i1 [[TMP19]], <4 x i32> [[WIDE_LOAD11]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP31]] = select i1 [[TMP19]], <4 x i32> [[WIDE_LOAD10]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-VF4IC4-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP29]], <4 x i1> [[TMP25]], i32 [[TMP8]])
+; CHECK-VF4IC4-NEXT: [[TMP35:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP30]], <4 x i1> [[TMP26]], i32 [[TMP34]])
+; CHECK-VF4IC4-NEXT: [[TMP36:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP31]], <4 x i1> [[TMP32]], i32 [[TMP35]])
; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-VF4IC4: [[SCALAR_PH]]:
; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP36]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
; CHECK-VF4IC4-NEXT: br label %[[LOOP:.*]]
; CHECK-VF4IC4: [[LOOP]]:
; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -1224,7 +1260,7 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF4IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-VF4IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-VF4IC4: [[EXIT]]:
-; CHECK-VF4IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP36]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF4IC4-NEXT: ret i32 [[SEL_LCSSA]]
;
; CHECK-VF1IC4-LABEL: define i32 @select_variant_i32_from_icmp(
>From 33fce63b094042a08ba04a1771c11f60027dd83d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 2 Mar 2026 11:58:15 +0000
Subject: [PATCH 2/2] Rebase tests
---
.../conditional-scalar-assignment-vplan.ll | 4 +-
.../X86/conditional-scalar-assignment.ll | 44 +-
.../iv-select-cmp-non-const-iv-start.ll | 84 ++-
.../LoopVectorize/iv-select-cmp-trunc.ll | 536 ++++++++++++++++--
.../Transforms/LoopVectorize/select-cmp.ll | 83 ++-
5 files changed, 673 insertions(+), 78 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index 81855a314945d..a270993075b4a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -112,7 +112,7 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP12]]>, vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP13]]>, vp<[[VP12]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
@@ -280,7 +280,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP14]]>, vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
index 3d0905081e2a5..93c66b3acfd90 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
@@ -757,10 +757,10 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; X86-LABEL: define i32 @simple_csa_int_select_use_interleave(
; X86-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) {
; X86-NEXT: [[ENTRY:.*]]:
-; X86-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; X86-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; X86-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; X86: [[VECTOR_PH]]:
-; X86-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; X86-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
; X86-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; X86-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
; X86-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -768,24 +768,34 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; X86: [[LOOP]]:
; X86-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; X86-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[LOOP]] ]
; X86-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; X86-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[LOOP]] ]
; X86-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
+; X86-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[LD_ADDR]], i64 4
; X86-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[LD_ADDR]], align 4
+; X86-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
; X86-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; X86-NEXT: [[TMP10:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD2]]
; X86-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; X86-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; X86-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP10]]
+; X86-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP3]], [[TMP12]]
+; X86-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
; X86-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; X86-NEXT: [[TMP11]] = select i1 [[TMP4]], <4 x i1> [[TMP10]], <4 x i1> [[TMP1]]
; X86-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
-; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; X86-NEXT: [[TMP13]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[VEC_PHI1]]
+; X86-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; X86-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; X86-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; X86: [[MIDDLE_BLOCK]]:
; X86-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP6]], <4 x i1> [[TMP5]], i32 -1)
+; X86-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP13]], <4 x i1> [[TMP11]], i32 [[TMP8]])
; X86-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; X86-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; X86: [[SCALAR_PH]]:
; X86-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; X86-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; X86-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; X86-NEXT: br label %[[LOOP1:.*]]
; X86: [[LOOP1]]:
; X86-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
@@ -798,16 +808,16 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; X86-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; X86-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; X86: [[EXIT]]:
-; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; X86-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP1]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; X86-NEXT: ret i32 [[SELECT_DATA_LCSSA]]
;
; AVX512-LABEL: define i32 @simple_csa_int_select_use_interleave(
; AVX512-SAME: i64 [[N:%.*]], ptr [[DATA:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
; AVX512-NEXT: [[ENTRY:.*]]:
-; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; AVX512: [[VECTOR_PH]]:
-; AVX512-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; AVX512-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
; AVX512-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; AVX512-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[A]], i64 0
; AVX512-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
@@ -815,24 +825,34 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
; AVX512-NEXT: [[TMP0:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[TMP9:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; AVX512-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
+; AVX512-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 16
; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
+; AVX512-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i32>, ptr [[TMP10]], align 4
; AVX512-NEXT: [[TMP2:%.*]] = icmp slt <16 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; AVX512-NEXT: [[TMP12:%.*]] = icmp slt <16 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD2]]
; AVX512-NEXT: [[TMP3:%.*]] = freeze <16 x i1> [[TMP2]]
-; AVX512-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
+; AVX512-NEXT: [[TMP14:%.*]] = freeze <16 x i1> [[TMP12]]
+; AVX512-NEXT: [[TMP15:%.*]] = or <16 x i1> [[TMP3]], [[TMP14]]
+; AVX512-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP15]])
; AVX512-NEXT: [[TMP5]] = select i1 [[TMP4]], <16 x i1> [[TMP2]], <16 x i1> [[TMP0]]
+; AVX512-NEXT: [[TMP11]] = select i1 [[TMP4]], <16 x i1> [[TMP12]], <16 x i1> [[TMP9]]
; AVX512-NEXT: [[TMP6]] = select i1 [[TMP4]], <16 x i32> [[WIDE_LOAD]], <16 x i32> [[VEC_PHI]]
-; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; AVX512-NEXT: [[TMP13]] = select i1 [[TMP4]], <16 x i32> [[WIDE_LOAD2]], <16 x i32> [[VEC_PHI1]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; AVX512-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; AVX512-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; AVX512: [[MIDDLE_BLOCK]]:
; AVX512-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v16i32(<16 x i32> [[TMP6]], <16 x i1> [[TMP5]], i32 -1)
+; AVX512-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v16i32(<16 x i32> [[TMP13]], <16 x i1> [[TMP11]], i32 [[TMP8]])
; AVX512-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; AVX512-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; AVX512: [[SCALAR_PH]]:
; AVX512-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; AVX512-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; AVX512-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
; AVX512-NEXT: br label %[[LOOP:.*]]
; AVX512: [[LOOP]]:
; AVX512-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -845,7 +865,7 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; AVX512-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; AVX512-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; AVX512: [[EXIT]]:
-; AVX512-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; AVX512-NEXT: [[SELECT_DATA_LCSSA:%.*]] = phi i32 [ [[SELECT_DATA]], %[[LOOP]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
; AVX512-NEXT: ret i32 [[SELECT_DATA_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
index 0bc5987656c7a..7effcb278ceb6 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
@@ -429,20 +429,88 @@ define i32 @select_trunc_non_const_iv_start_signed_guard(ptr %a, i32 %rdx_start,
; CHECK-VF1IC4: [[FOR_BODY_PREHEADER]]:
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = sext i32 [[IV_START]] to i64
; CHECK-VF1IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = sext i32 [[N]] to i64
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[TMP0]]
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP15]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP15]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP15]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], [[N_VEC]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX_07:%.*]] = phi i32 [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[RDX_START]], %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[RDX_START]], %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ [[RDX_START]], %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ [[RDX_START]], %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP37:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[OFFSET_IDX4:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = trunc i64 [[OFFSET_IDX4]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], 0
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = add i32 [[TMP10]], 1
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = add i32 [[TMP10]], 2
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = add i32 [[TMP10]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 4
; CHECK-VF1IC4-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP1]], 3
-; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP1]], i32 [[TMP2]], i32 [[RDX_07]]
-; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = icmp sgt i32 [[TMP20]], 3
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = icmp sgt i32 [[TMP21]], 3
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp sgt i32 [[TMP22]], 3
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[CMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
+; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[CMP1]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP33]], i1 [[TMP25]], i1 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP33]], i1 [[TMP26]], i1 [[TMP6]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP33]], i32 [[TMP11]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP33]], i32 [[TMP12]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP33]], i32 [[TMP13]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP41]] = select i1 [[TMP33]], i32 [[TMP14]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[RDX_START]]
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP43]]
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = select i1 [[TMP36]], i32 [[TMP40]], i32 [[TMP44]]
+; CHECK-VF1IC4-NEXT: [[TMP46:%.*]] = select i1 [[TMP37]], i32 [[TMP41]], i32 [[TMP45]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP15]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP46]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[TMP47]], 3
+; CHECK-VF1IC4-NEXT: [[TMP48:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP2]], i32 [[TMP48]], i32 [[RDX_07]]
+; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-VF1IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY1]] ], [ [[TMP46]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: br label %[[EXIT]]
; CHECK-VF1IC4: [[EXIT]]:
; CHECK-VF1IC4-NEXT: [[IDX_0_LCSSA:%.*]] = phi i32 [ [[RDX_START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -492,4 +560,6 @@ exit:
; CHECK-VF1IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK-VF1IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK-VF1IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index 4381ac4ff3cfc..4166c328835c2 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -831,20 +831,84 @@ define i32 @select_icmp_const_truncated_iv_unwidened_exit(ptr %a, i64 %n) {
; CHECK-VF1IC4-NEXT: [[CMP_SGT:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-VF1IC4-NEXT: br i1 [[CMP_SGT]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-VF1IC4: [[FOR_BODY_PREHEADER]]:
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP37:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i32 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 3
-; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = icmp sgt i32 [[TMP17]], 3
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = icmp sgt i32 [[TMP18]], 3
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP30]], i1 [[TMP22]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP30]], i1 [[TMP23]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP30]], i32 [[TMP8]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP30]], i32 [[TMP9]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP30]], i32 [[TMP10]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP30]], i32 [[TMP11]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP40:%.*]] = select i1 [[TMP31]], i32 [[TMP35]], i32 331
+; CHECK-VF1IC4-NEXT: [[TMP41:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP40]]
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = select i1 [[TMP33]], i32 [[TMP37]], i32 [[TMP41]]
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP42]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[SPEC_SELECT:%.*]], %[[FOR_BODY1]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP44]], 3
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP1]], i32 [[TMP45]], i32 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-VF1IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY1]] ], [ [[TMP43]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: br label %[[EXIT]]
; CHECK-VF1IC4: [[EXIT]]:
; CHECK-VF1IC4-NEXT: [[RDX_LCSSA:%.*]] = phi i32 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1031,20 +1095,84 @@ define i32 @select_icmp_const_truncated_iv_unsigned_loop_guard(ptr %a, i32 %n) {
; CHECK-VF1IC4-NEXT: br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER:.*]]
; CHECK-VF1IC4: [[FOR_BODY_PREHEADER]]:
; CHECK-VF1IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ 331, %[[FOR_BODY_PREHEADER]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP37:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i32 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 3
-; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP1]], i32 [[TMP1]], i32 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = icmp sgt i32 [[TMP17]], 3
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = icmp sgt i32 [[TMP18]], 3
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP1]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP30]], i1 [[TMP22]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP30]], i1 [[TMP23]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP30]], i32 [[TMP8]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP30]], i32 [[TMP9]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP30]], i32 [[TMP10]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP30]], i32 [[TMP11]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP40:%.*]] = select i1 [[TMP31]], i32 [[TMP35]], i32 331
+; CHECK-VF1IC4-NEXT: [[TMP41:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP40]]
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = select i1 [[TMP33]], i32 [[TMP37]], i32 [[TMP41]]
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP42]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 331, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[TMP44]], 3
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP2]], i32 [[TMP45]], i32 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-VF1IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY1]] ], [ [[TMP43]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: br label %[[EXIT]]
; CHECK-VF1IC4: [[EXIT]]:
; CHECK-VF1IC4-NEXT: [[RDX_LCSSA:%.*]] = phi i32 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1176,21 +1304,67 @@ define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(ptr %a) {
;
; CHECK-VF1IC4-LABEL: define i32 @not_vectorized_select_icmp_truncated_iv_out_of_bound(
; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT: [[ENTRY:.*:]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 4294967294, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4: [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 331, %[[FOR_BODY]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 331, %[[FOR_BODY]] ], [ [[TMP36:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 331, %[[FOR_BODY]] ], [ [[TMP37:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 331, %[[FOR_BODY]] ], [ [[TMP38:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = add i64 4294967294, [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[OFFSET_IDX4:%.*]] = add i64 4294967294, [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[OFFSET_IDX4]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i32 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 3
-; CHECK-VF1IC4-NEXT: [[CONV:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP]], i32 [[CONV]], i32 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
-; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 9223372036854775806
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = icmp sgt i32 [[TMP17]], 3
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = icmp sgt i32 [[TMP18]], 3
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp sgt i32 [[TMP19]], 3
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP30]], i1 [[TMP22]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP30]], i1 [[TMP23]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP30]], i32 [[TMP8]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP30]], i32 [[TMP9]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP30]], i32 [[TMP10]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP30]], i32 [[TMP11]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9223372032559808512
+; CHECK-VF1IC4-NEXT: br i1 [[TMP39]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-VF1IC4: [[EXIT]]:
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP40:%.*]] = select i1 [[TMP31]], i32 [[TMP35]], i32 331
+; CHECK-VF1IC4-NEXT: [[TMP41:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP40]]
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = select i1 [[TMP33]], i32 [[TMP37]], i32 [[TMP41]]
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP42]]
+; CHECK-VF1IC4-NEXT: br label %[[EXIT1:.*]]
+; CHECK-VF1IC4: [[EXIT1]]:
; CHECK-VF1IC4-NEXT: ret i32 [[SPEC_SELECT_LCSSA]]
;
entry:
@@ -1370,22 +1544,94 @@ define i32 @not_vectorized_select_iv_icmp_no_guard(ptr %a, ptr %b, i32 %start, i
; CHECK-VF1IC4-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[START:%.*]], i32 [[N:%.*]]) {
; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
; CHECK-VF1IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP44:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP45:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP46:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i32 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP23]], align 4
; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
-; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = icmp sgt i32 [[TMP17]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = icmp sgt i32 [[TMP18]], [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP]]
+; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
+; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
+; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP41]] = select i1 [[TMP38]], i1 [[TMP30]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP42]] = select i1 [[TMP38]], i1 [[TMP31]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP43]] = select i1 [[TMP38]], i32 [[TMP8]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP44]] = select i1 [[TMP38]], i32 [[TMP9]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP45]] = select i1 [[TMP38]], i32 [[TMP10]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP46]] = select i1 [[TMP38]], i32 [[TMP11]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP48:%.*]] = select i1 [[TMP39]], i32 [[TMP43]], i32 [[START]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = select i1 [[TMP40]], i32 [[TMP44]], i32 [[TMP48]]
+; CHECK-VF1IC4-NEXT: [[TMP50:%.*]] = select i1 [[TMP41]], i32 [[TMP45]], i32 [[TMP49]]
+; CHECK-VF1IC4-NEXT: [[TMP51:%.*]] = select i1 [[TMP42]], i32 [[TMP46]], i32 [[TMP50]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP51]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP52:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP53:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP52]], [[TMP53]]
+; CHECK-VF1IC4-NEXT: [[TMP54:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP1]], i32 [[TMP54]], i32 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-VF1IC4: [[EXIT]]:
-; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY1]] ], [ [[TMP51]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: ret i32 [[COND_LCSSA]]
;
entry:
@@ -1534,21 +1780,79 @@ define i32 @not_vectorized_select_fcmp_invalid_const_ub(ptr %a) {
;
; CHECK-VF1IC4-LABEL: define i32 @not_vectorized_select_fcmp_invalid_const_ub(
; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT: [[ENTRY:.*:]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ -1, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4: [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -1, %[[FOR_BODY]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -1, %[[FOR_BODY]] ], [ [[TMP36:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ -1, %[[FOR_BODY]] ], [ [[TMP37:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ -1, %[[FOR_BODY]] ], [ [[TMP38:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[FOR_BODY]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i32 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load float, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load float, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = fcmp fast olt float [[TMP0]], 0.000000e+00
-; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = trunc i64 [[IV]] to i32
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = fcmp fast olt float [[TMP17]], 0.000000e+00
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = fcmp fast olt float [[TMP18]], 0.000000e+00
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = fcmp fast olt float [[TMP19]], 0.000000e+00
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = freeze i1 [[CMP]]
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP21]]
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = or i1 [[TMP24]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP22]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = or i1 [[TMP26]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = or i1 [[TMP28]], [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP30]], i1 [[CMP]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP30]], i1 [[TMP21]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP30]], i1 [[TMP22]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP30]], i1 [[TMP23]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP30]], i32 [[TMP8]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP30]], i32 [[TMP9]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP30]], i32 [[TMP10]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP30]], i32 [[TMP11]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2147483648
+; CHECK-VF1IC4-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP40:%.*]] = select i1 [[TMP31]], i32 [[TMP35]], i32 -1
+; CHECK-VF1IC4-NEXT: [[TMP41:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP40]]
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = select i1 [[TMP33]], i32 [[TMP37]], i32 [[TMP41]]
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP42]]
+; CHECK-VF1IC4-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ 2147483648, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[TMP43]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP1:%.*]] = fcmp fast olt float [[TMP44]], 0.000000e+00
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = trunc i64 [[IV1]] to i32
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT]] = select i1 [[CMP1]], i32 [[TMP45]], i32 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 2147483649
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-VF1IC4: [[EXIT]]:
-; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY1]] ]
; CHECK-VF1IC4-NEXT: ret i32 [[SPEC_SELECT_LCSSA]]
;
entry:
@@ -1744,22 +2048,94 @@ define i16 @not_vectorized_select_iv_icmp_overflow_unwidened_tripcount(ptr %a, p
; CHECK-VF1IC4-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-VF1IC4: [[FOR_BODY_PREHEADER]]:
; CHECK-VF1IC4-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-VF1IC4: [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i16 [ [[START]], %[[FOR_BODY_PREHEADER]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i16 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i16 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP44:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i16 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP45:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i16 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP46:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = trunc i64 [[IV]] to i16
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i16 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = add i16 [[TMP7]], 1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i16 [[TMP7]], 2
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i16 [[TMP7]], 3
; CHECK-VF1IC4-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 4
; CHECK-VF1IC4-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP23]], align 4
; CHECK-VF1IC4-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
-; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = trunc i64 [[IV]] to i16
-; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP3]], i16 [[TMP2]], i16 [[RDX]]
-; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = icmp sgt i32 [[TMP17]], [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = icmp sgt i32 [[TMP18]], [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = icmp sgt i32 [[TMP19]], [[TMP27]]
+; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[CMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP29]]
+; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = or i1 [[TMP32]], [[TMP33]]
+; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP30]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = or i1 [[TMP34]], [[TMP35]]
+; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = freeze i1 [[TMP31]]
+; CHECK-VF1IC4-NEXT: [[TMP38:%.*]] = or i1 [[TMP36]], [[TMP37]]
+; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP38]], i1 [[CMP3]], i1 [[TMP12]]
+; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP38]], i1 [[TMP29]], i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP41]] = select i1 [[TMP38]], i1 [[TMP30]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP42]] = select i1 [[TMP38]], i1 [[TMP31]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP43]] = select i1 [[TMP38]], i16 [[TMP8]], i16 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP44]] = select i1 [[TMP38]], i16 [[TMP9]], i16 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP45]] = select i1 [[TMP38]], i16 [[TMP10]], i16 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP46]] = select i1 [[TMP38]], i16 [[TMP11]], i16 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP48:%.*]] = select i1 [[TMP39]], i16 [[TMP43]], i16 [[START]]
+; CHECK-VF1IC4-NEXT: [[TMP49:%.*]] = select i1 [[TMP40]], i16 [[TMP44]], i16 [[TMP48]]
+; CHECK-VF1IC4-NEXT: [[TMP50:%.*]] = select i1 [[TMP41]], i16 [[TMP45]], i16 [[TMP49]]
+; CHECK-VF1IC4-NEXT: [[TMP51:%.*]] = select i1 [[TMP42]], i16 [[TMP46]], i16 [[TMP50]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP51]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK-VF1IC4: [[FOR_BODY1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY1]] ]
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP52:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-VF1IC4-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[TMP53:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[TMP52]], [[TMP53]]
+; CHECK-VF1IC4-NEXT: [[TMP54:%.*]] = trunc i64 [[IV1]] to i16
+; CHECK-VF1IC4-NEXT: [[COND]] = select i1 [[CMP4]], i16 [[TMP54]], i16 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[INC]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[WIDE_TRIP_COUNT]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK-VF1IC4: [[EXIT_LOOPEXIT]]:
-; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i16 [ [[COND]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[COND_LCSSA:%.*]] = phi i16 [ [[COND]], %[[FOR_BODY1]] ], [ [[TMP51]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: br label %[[EXIT]]
; CHECK-VF1IC4: [[EXIT]]:
; CHECK-VF1IC4-NEXT: [[RDX_0_LCSSA:%.*]] = phi i16 [ [[START]], %[[ENTRY]] ], [ [[COND_LCSSA]], %[[EXIT_LOOPEXIT]] ]
@@ -1955,20 +2331,80 @@ define i1 @select_with_trunc_i1_iv(i64 %n, i64 %start) {
; CHECK-VF1IC4-LABEL: define i1 @select_with_trunc_i1_iv(
; CHECK-VF1IC4-SAME: i64 [[N:%.*]], i64 [[START:%.*]]) {
; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = add i64 [[START]], [[N_VEC]]
; CHECK-VF1IC4-NEXT: br label %[[LOOP:.*]]
; CHECK-VF1IC4: [[LOOP]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC4-NEXT: [[CTR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[CTR_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC4-NEXT: [[ACCUM:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = trunc i64 [[INDEX]] to i1
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = add i1 [[TMP9]], false
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = add i1 [[TMP9]], true
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = add i1 [[TMP9]], false
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = add i1 [[TMP9]], true
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = icmp ne i64 [[OFFSET_IDX]], 0
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = freeze i1 [[TMP14]]
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP15]]
+; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = freeze i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = freeze i1 [[TMP17]]
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = or i1 [[TMP22]], [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP25]] = select i1 [[TMP24]], i1 [[TMP14]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP26]] = select i1 [[TMP24]], i1 [[TMP15]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP27]] = select i1 [[TMP24]], i1 [[TMP16]], i1 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP28]] = select i1 [[TMP24]], i1 [[TMP17]], i1 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP29]] = select i1 [[TMP24]], i1 [[TMP10]], i1 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP30]] = select i1 [[TMP24]], i1 [[TMP11]], i1 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP31]] = select i1 [[TMP24]], i1 [[TMP12]], i1 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP24]], i1 [[TMP13]], i1 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP34:%.*]] = select i1 [[TMP25]], i1 [[TMP29]], i1 false
+; CHECK-VF1IC4-NEXT: [[TMP35:%.*]] = select i1 [[TMP26]], i1 [[TMP30]], i1 [[TMP34]]
+; CHECK-VF1IC4-NEXT: [[TMP36:%.*]] = select i1 [[TMP27]], i1 [[TMP31]], i1 [[TMP35]]
+; CHECK-VF1IC4-NEXT: [[TMP37:%.*]] = select i1 [[TMP28]], i1 [[TMP32]], i1 [[TMP36]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ false, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[LOOP1:.*]]
+; CHECK-VF1IC4: [[LOOP1]]:
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-VF1IC4-NEXT: [[CTR:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[CTR_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-VF1IC4-NEXT: [[ACCUM:%.*]] = phi i1 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP1]] ]
; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 0
; CHECK-VF1IC4-NEXT: [[TRUNC:%.*]] = trunc i64 [[CTR]] to i1
; CHECK-VF1IC4-NEXT: [[SEL]] = select i1 [[CMP]], i1 [[ACCUM]], i1 [[TRUNC]]
; CHECK-VF1IC4-NEXT: [[CTR_NEXT]] = add i64 [[CTR]], 1
; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-VF1IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[N]], [[CTR]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK-VF1IC4: [[EXIT]]:
-; CHECK-VF1IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], %[[LOOP1]] ], [ [[TMP37]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: ret i1 [[SEL_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index a7f12136c4ae0..af388a2503bce 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -1266,21 +1266,88 @@ define i32 @select_variant_i32_from_icmp(ptr %v1, ptr %v2, i64 %n) {
; CHECK-VF1IC4-LABEL: define i32 @select_variant_i32_from_icmp(
; CHECK-VF1IC4-SAME: ptr [[V1:%.*]], ptr [[V2:%.*]], i64 [[N:%.*]]) {
; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-VF1IC4-NEXT: br label %[[LOOP:.*]]
; CHECK-VF1IC4: [[LOOP]]:
-; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ 3, %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP37:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 2
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = add i64 [[IV]], 3
; CHECK-VF1IC4-NEXT: [[GEP_V1_IV:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[LOAD_V1_IV:%.*]] = load i32, ptr [[GEP_V1_IV]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP10]], align 4
; CHECK-VF1IC4-NEXT: [[GEP_V2_IV:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[TMP5]]
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[TMP6]]
; CHECK-VF1IC4-NEXT: [[LOAD_V2_IV:%.*]] = load i32, ptr [[GEP_V2_IV]], align 4
-; CHECK-VF1IC4-NEXT: [[CMP_V1_IV_3:%.*]] = icmp eq i32 [[LOAD_V1_IV]], 3
-; CHECK-VF1IC4-NEXT: [[SEL]] = select i1 [[CMP_V1_IV_3]], i32 [[RDX]], i32 [[LOAD_V2_IV]]
-; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = icmp ne i32 [[LOAD_V1_IV]], 3
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = icmp ne i32 [[TMP12]], 3
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = icmp ne i32 [[TMP13]], 3
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = icmp ne i32 [[TMP14]], 3
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP23]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP24]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP25]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
+; CHECK-VF1IC4-NEXT: [[TMP32:%.*]] = freeze i1 [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP33:%.*]] = or i1 [[TMP31]], [[TMP32]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP33]], i1 [[TMP23]], i1 [[TMP0]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP33]], i1 [[TMP24]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP33]], i1 [[TMP25]], i1 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP33]], i1 [[TMP26]], i1 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP33]], i32 [[LOAD_V2_IV]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP33]], i32 [[TMP20]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP40]] = select i1 [[TMP33]], i32 [[TMP21]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP41]] = select i1 [[TMP33]], i32 [[TMP22]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 3
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP43]]
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = select i1 [[TMP36]], i32 [[TMP40]], i32 [[TMP44]]
+; CHECK-VF1IC4-NEXT: [[TMP46:%.*]] = select i1 [[TMP37]], i32 [[TMP41]], i32 [[TMP45]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP46]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[LOOP1:.*]]
+; CHECK-VF1IC4: [[LOOP1]]:
+; CHECK-VF1IC4-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP1]] ]
+; CHECK-VF1IC4-NEXT: [[GEP_V1_IV1:%.*]] = getelementptr inbounds i32, ptr [[V1]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[LOAD_V1_IV1:%.*]] = load i32, ptr [[GEP_V1_IV1]], align 4
+; CHECK-VF1IC4-NEXT: [[GEP_V2_IV1:%.*]] = getelementptr inbounds i32, ptr [[V2]], i64 [[IV1]]
+; CHECK-VF1IC4-NEXT: [[LOAD_V2_IV1:%.*]] = load i32, ptr [[GEP_V2_IV1]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP_V1_IV_3:%.*]] = icmp eq i32 [[LOAD_V1_IV1]], 3
+; CHECK-VF1IC4-NEXT: [[SEL]] = select i1 [[CMP_V1_IV_3]], i32 [[RDX]], i32 [[LOAD_V2_IV1]]
+; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-VF1IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-VF1IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-VF1IC4: [[EXIT]]:
-; CHECK-VF1IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP1]] ], [ [[TMP46]], %[[MIDDLE_BLOCK]] ]
; CHECK-VF1IC4-NEXT: ret i32 [[SEL_LCSSA]]
;
entry:
@@ -1387,4 +1454,6 @@ exit: ; preds = %loop
; CHECK-VF1IC4: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
; CHECK-VF1IC4: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
; CHECK-VF1IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; CHECK-VF1IC4: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK-VF1IC4: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
;.
More information about the llvm-commits
mailing list