[llvm] [LV] Add initial cost model for VPScalarIVSteps (PR #203347)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 05:01:25 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/203347
>From d35c65ef3bf085c61ced7a6c6abae1badb6ff657 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 12 Jun 2026 09:55:54 +0000
Subject: [PATCH 1/4] Add test
---
.../AArch64/scalar-steps-cost.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
new file mode 100644
index 0000000000000..aa448cdd6c689
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
@@ -0,0 +1,51 @@
+; REQUIRES: asserts
+; RUN: opt -p loop-vectorize -mtriple=arm64-apple-macosx -S -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+define i32 @scalar_steps_all_lanes(ptr %start, ptr %end) {
+; CHECK-LABEL: LV: Checking a loop in 'scalar_steps_all_lanes'
+; CHECK: Cost of 0 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 0 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+entry:
+ br label %loop
+
+loop:
+ %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
+ %l = load i32, ptr %iv, align 1
+ %1 = and i32 %l, 1
+ %rdx.next = or i32 %rdx, %1
+ %iv.next = getelementptr i8, ptr %iv, i64 1
+ %ec = icmp ult ptr %iv, %end
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret i32 %rdx.next
+}
+
+define void @scalar_steps_first_lane_only(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: LV: Checking a loop in 'scalar_steps_first_lane_only'
+; CHECK: Cost of 0 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 0 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep1 = getelementptr inbounds i32, ptr %b, i64 %iv
+ %ld1 = load i32, ptr %gep1
+ %gep2 = getelementptr inbounds i32, ptr %c, i64 %iv
+ %ld2 = load i32, ptr %gep2
+ %add = add nsw i32 %ld2, %ld1
+ %gep3 = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %add, ptr %gep3
+ %iv.next = add i64 %iv, 1
+ %iv.trunc = trunc i64 %iv.next to i32
+ %exitcond = icmp eq i32 %iv.trunc, 128
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
>From 4465924bd85bdae8e040839c5a66bdccc93cb486 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 12 Jun 2026 09:56:11 +0000
Subject: [PATCH 2/4] [LV] Add initial cost model for VPScalarIVSteps
This PR currently only adds a cost model for integer types in
non-replicating regions in order to limit the scope of impact.
We can also support replicating regions, but that requires
looking for a recipe with an underlying value in the same
region in order to get a BasicBlock to pass in to the
getPredBlockCostDivisor function. This can be done in a future
PR.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 5 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 33 +
.../LoopVectorize/AArch64/induction-costs.ll | 249 +++-----
.../replicating-load-store-costs-apple.ll | 167 ++---
.../AArch64/replicating-load-store-costs.ll | 85 +--
.../AArch64/scalar-steps-cost.ll | 4 +-
.../ARM/mve-reg-pressure-spills.ll | 4 +-
.../LoopVectorize/RISCV/uniform-load-store.ll | 21 +-
.../WebAssembly/memory-interleave.ll | 62 +-
.../LoopVectorize/X86/cost-model.ll | 217 ++-----
.../X86/fixed-order-recurrence.ll | 32 +-
.../LoopVectorize/X86/interleave-cost.ll | 148 +----
.../X86/replicating-load-store-costs.ll | 597 ++++++------------
.../LoopVectorize/X86/strided_load_cost.ll | 108 +---
.../X86/vector_ptr_load_store.ll | 2 +-
15 files changed, 512 insertions(+), 1222 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 16f3c049e6702..857a9b2df7181 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4288,10 +4288,7 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
/// Return the cost of this VPScalarIVStepsRecipe.
InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override {
- // TODO: Compute accurate cost after retiring the legacy cost model.
- return 0;
- }
+ VPCostContext &Ctx) const override;
VPValue *getStepValue() const { return getOperand(1); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 17b90c733a817..3da35a49188a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2960,6 +2960,39 @@ void VPDerivedIVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
+InstructionCost VPScalarIVStepsRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ // TODO: Add costs for floating point.
+ Type *BaseIVTy = getOperand(0)->getScalarType();
+ if (!BaseIVTy->isIntegerTy())
+ return 0;
+
+ // TODO: Add support for predicated regions. Requires scaling the cost by the
+ // probability of entering the block.
+ if (getRegion() && getRegion()->isReplicator())
+ return 0;
+
+ // Typically the operations are:
+ // 1. Add the start index to each lane value.
+ // 2. Multiply the start index by the step.
+ // 3. Add the scaled start index to base IV.
+ // Any code generated for 1 and 2 should be loop invariant and therefore
+ // hoisted out of the loop. We only need to add on the cost of 3.
+
+ // Determine the number of scalar adds we need to generate.
+ const unsigned NumLanes =
+ vputils::onlyFirstLaneUsed(this) ? 1 : VF.getKnownMinValue();
+
+ // If there is no start index the first lane will be free, since
+ // add i32 x, 0 -> i32 x
+ // However, the start index is only well defined after the unroll
+ // transformation that sets it. For now just assume it is 0, which means
+ // (NumLanes - 1) add operations.
+ const unsigned NumAdds = NumLanes - 1;
+ return NumAdds * Ctx.TTI.getArithmeticInstrCost(Instruction::Add, BaseIVTy,
+ Ctx.CostKind);
+}
+
void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
// Fast-math-flags propagate from the original induction instruction.
IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index d6d10a3956fc4..bc7ee73c28775 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -424,75 +424,48 @@ define i32 @load_from_pointer_induction(ptr %start, ptr %end) {
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[END1]], i64 [[START2]])
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 6
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 7
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP3]]
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]]
-; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP7]]
-; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
-; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[NEXT_GEP4]], align 1
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[NEXT_GEP5]], align 1
-; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP6]], align 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 1
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 2
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP13]], i32 3
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[NEXT_GEP7]], align 1
-; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[NEXT_GEP8]], align 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[NEXT_GEP9]], align 1
-; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[NEXT_GEP10]], align 1
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i32 0
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i32 1
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i32 2
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i32 3
-; CHECK-NEXT: [[TMP26:%.*]] = and <4 x i32> [[TMP17]], splat (i32 1)
-; CHECK-NEXT: [[TMP27:%.*]] = and <4 x i32> [[TMP25]], splat (i32 1)
-; CHECK-NEXT: [[TMP28]] = or <4 x i32> [[VEC_PHI]], [[TMP26]]
-; CHECK-NEXT: [[TMP29]] = or <4 x i32> [[VEC_PHI3]], [[TMP27]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP4]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP4]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[TMP5]], 1
+; CHECK-NEXT: [[TMP8]] = or i32 [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[TMP9]] = or i32 [[VEC_PHI3]], [[TMP7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i32> [[TMP29]], [[TMP28]]
-; CHECK-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = or i32 [[TMP9]], [[TMP8]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[IV]], align 1
-; CHECK-NEXT: [[TMP32:%.*]] = and i32 [[L]], 1
-; CHECK-NEXT: [[RDX_NEXT]] = or i32 [[RDX]], [[TMP32]]
+; CHECK-NEXT: [[TMP11:%.*]] = and i32 [[L]], 1
+; CHECK-NEXT: [[RDX_NEXT]] = or i32 [[RDX]], [[TMP11]]
; CHECK-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 1
; CHECK-NEXT: [[EC:%.*]] = icmp ult ptr [[IV]], [[END]]
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RDX_NEXT_LCSSA]]
;
entry:
@@ -757,13 +730,10 @@ exit:
define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr noalias %dst) {
; CHECK-LABEL: define i64 @live_out_extract_from_ptr_iv_increment(
; CHECK-SAME: i64 [[COUNT:%.*]], ptr [[START:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[COUNT]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -789,37 +759,37 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP3]], 42
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP3]], 45
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP3]]
-; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]]
-; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP7]]
-; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
-; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]]
-; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]]
-; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
-; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
-; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
-; CHECK-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP14]]
-; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
-; CHECK-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP16]]
-; CHECK-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
-; CHECK-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP18]]
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]]
+; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
+; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP7]]
+; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
+; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]]
+; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]]
+; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
+; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
+; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
+; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP14]]
+; CHECK-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
+; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP16]]
+; CHECK-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
+; CHECK-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[NEXT_GEP2]], align 1
-; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[NEXT_GEP3]], align 1
-; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[NEXT_GEP4]], align 1
-; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[NEXT_GEP5]], align 1
-; CHECK-NEXT: [[TMP24:%.*]] = load i8, ptr [[NEXT_GEP6]], align 1
-; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[NEXT_GEP7]], align 1
-; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr [[NEXT_GEP8]], align 1
-; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[NEXT_GEP9]], align 1
-; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[NEXT_GEP10]], align 1
-; CHECK-NEXT: [[TMP29:%.*]] = load i8, ptr [[NEXT_GEP11]], align 1
-; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[NEXT_GEP12]], align 1
-; CHECK-NEXT: [[TMP31:%.*]] = load i8, ptr [[NEXT_GEP13]], align 1
-; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[NEXT_GEP14]], align 1
-; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[NEXT_GEP15]], align 1
-; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[NEXT_GEP16]], align 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[NEXT_GEP1]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[NEXT_GEP2]], align 1
+; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[NEXT_GEP3]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[NEXT_GEP4]], align 1
+; CHECK-NEXT: [[TMP24:%.*]] = load i8, ptr [[NEXT_GEP5]], align 1
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[NEXT_GEP6]], align 1
+; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr [[NEXT_GEP7]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[NEXT_GEP8]], align 1
+; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[NEXT_GEP9]], align 1
+; CHECK-NEXT: [[TMP29:%.*]] = load i8, ptr [[NEXT_GEP10]], align 1
+; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[NEXT_GEP11]], align 1
+; CHECK-NEXT: [[TMP31:%.*]] = load i8, ptr [[NEXT_GEP12]], align 1
+; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[NEXT_GEP13]], align 1
+; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[NEXT_GEP14]], align 1
+; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[NEXT_GEP15]], align 1
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <16 x i8> poison, i8 [[TMP19]], i32 0
; CHECK-NEXT: [[TMP36:%.*]] = insertelement <16 x i8> [[TMP35]], i8 [[TMP20]], i32 1
; CHECK-NEXT: [[TMP37:%.*]] = insertelement <16 x i8> [[TMP36]], i8 [[TMP21]], i32 2
@@ -840,21 +810,21 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[TMP52:%.*]] = extractelement <16 x i32> [[TMP51]], i64 15
; CHECK-NEXT: store i32 [[TMP52]], ptr [[DST]], align 4
; CHECK-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 3
-; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 3
-; CHECK-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 3
-; CHECK-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 3
-; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 3
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 3
-; CHECK-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 3
-; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 3
-; CHECK-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 3
-; CHECK-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 3
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 3
-; CHECK-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 3
-; CHECK-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 3
-; CHECK-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 3
-; CHECK-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 3
-; CHECK-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 3
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 3
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 3
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 3
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 3
+; CHECK-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 3
+; CHECK-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 3
+; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 3
+; CHECK-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 3
+; CHECK-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 3
+; CHECK-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 3
+; CHECK-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 3
+; CHECK-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 3
+; CHECK-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 3
+; CHECK-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 3
+; CHECK-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 3
; CHECK-NEXT: [[TMP69:%.*]] = insertelement <16 x ptr> poison, ptr [[TMP53]], i32 0
; CHECK-NEXT: [[TMP70:%.*]] = insertelement <16 x ptr> [[TMP69]], ptr [[TMP54]], i32 1
; CHECK-NEXT: [[TMP71:%.*]] = insertelement <16 x ptr> [[TMP70]], ptr [[TMP55]], i32 2
@@ -878,85 +848,14 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[TMP86:%.*]] = ptrtoint <16 x ptr> [[TMP84]] to <16 x i64>
; CHECK-NEXT: [[TMP87:%.*]] = extractelement <16 x i64> [[TMP86]], i64 15
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8]]
-; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF17:%.*]] = urem i64 [[TMP0]], 8
-; CHECK-NEXT: [[N_VEC18:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF17]]
-; CHECK-NEXT: [[TMP88:%.*]] = mul i64 [[N_VEC18]], 3
-; CHECK-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP88]]
-; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX19:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT28:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP90:%.*]] = mul i64 [[INDEX19]], 3
-; CHECK-NEXT: [[TMP91:%.*]] = add i64 [[TMP90]], 3
-; CHECK-NEXT: [[TMP92:%.*]] = add i64 [[TMP90]], 6
-; CHECK-NEXT: [[TMP93:%.*]] = add i64 [[TMP90]], 9
-; CHECK-NEXT: [[TMP94:%.*]] = add i64 [[TMP90]], 12
-; CHECK-NEXT: [[TMP95:%.*]] = add i64 [[TMP90]], 15
-; CHECK-NEXT: [[TMP96:%.*]] = add i64 [[TMP90]], 18
-; CHECK-NEXT: [[TMP97:%.*]] = add i64 [[TMP90]], 21
-; CHECK-NEXT: [[NEXT_GEP20:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP90]]
-; CHECK-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP91]]
-; CHECK-NEXT: [[NEXT_GEP22:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP92]]
-; CHECK-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP93]]
-; CHECK-NEXT: [[NEXT_GEP24:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP94]]
-; CHECK-NEXT: [[NEXT_GEP25:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP95]]
-; CHECK-NEXT: [[NEXT_GEP26:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP96]]
-; CHECK-NEXT: [[NEXT_GEP27:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP97]]
-; CHECK-NEXT: [[TMP98:%.*]] = load i8, ptr [[NEXT_GEP20]], align 1
-; CHECK-NEXT: [[TMP99:%.*]] = load i8, ptr [[NEXT_GEP21]], align 1
-; CHECK-NEXT: [[TMP100:%.*]] = load i8, ptr [[NEXT_GEP22]], align 1
-; CHECK-NEXT: [[TMP101:%.*]] = load i8, ptr [[NEXT_GEP23]], align 1
-; CHECK-NEXT: [[TMP102:%.*]] = load i8, ptr [[NEXT_GEP24]], align 1
-; CHECK-NEXT: [[TMP103:%.*]] = load i8, ptr [[NEXT_GEP25]], align 1
-; CHECK-NEXT: [[TMP104:%.*]] = load i8, ptr [[NEXT_GEP26]], align 1
-; CHECK-NEXT: [[TMP105:%.*]] = load i8, ptr [[NEXT_GEP27]], align 1
-; CHECK-NEXT: [[TMP106:%.*]] = insertelement <8 x i8> poison, i8 [[TMP98]], i32 0
-; CHECK-NEXT: [[TMP107:%.*]] = insertelement <8 x i8> [[TMP106]], i8 [[TMP99]], i32 1
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <8 x i8> [[TMP107]], i8 [[TMP100]], i32 2
-; CHECK-NEXT: [[TMP109:%.*]] = insertelement <8 x i8> [[TMP108]], i8 [[TMP101]], i32 3
-; CHECK-NEXT: [[TMP110:%.*]] = insertelement <8 x i8> [[TMP109]], i8 [[TMP102]], i32 4
-; CHECK-NEXT: [[TMP111:%.*]] = insertelement <8 x i8> [[TMP110]], i8 [[TMP103]], i32 5
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <8 x i8> [[TMP111]], i8 [[TMP104]], i32 6
-; CHECK-NEXT: [[TMP113:%.*]] = insertelement <8 x i8> [[TMP112]], i8 [[TMP105]], i32 7
-; CHECK-NEXT: [[TMP114:%.*]] = zext <8 x i8> [[TMP113]] to <8 x i32>
-; CHECK-NEXT: [[TMP115:%.*]] = extractelement <8 x i32> [[TMP114]], i64 7
-; CHECK-NEXT: store i32 [[TMP115]], ptr [[DST]], align 4
-; CHECK-NEXT: [[TMP116:%.*]] = getelementptr i8, ptr [[NEXT_GEP20]], i64 3
-; CHECK-NEXT: [[TMP117:%.*]] = getelementptr i8, ptr [[NEXT_GEP21]], i64 3
-; CHECK-NEXT: [[TMP118:%.*]] = getelementptr i8, ptr [[NEXT_GEP22]], i64 3
-; CHECK-NEXT: [[TMP119:%.*]] = getelementptr i8, ptr [[NEXT_GEP23]], i64 3
-; CHECK-NEXT: [[TMP120:%.*]] = getelementptr i8, ptr [[NEXT_GEP24]], i64 3
-; CHECK-NEXT: [[TMP121:%.*]] = getelementptr i8, ptr [[NEXT_GEP25]], i64 3
-; CHECK-NEXT: [[TMP122:%.*]] = getelementptr i8, ptr [[NEXT_GEP26]], i64 3
-; CHECK-NEXT: [[TMP123:%.*]] = getelementptr i8, ptr [[NEXT_GEP27]], i64 3
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <8 x ptr> poison, ptr [[TMP116]], i32 0
-; CHECK-NEXT: [[TMP125:%.*]] = insertelement <8 x ptr> [[TMP124]], ptr [[TMP117]], i32 1
-; CHECK-NEXT: [[TMP126:%.*]] = insertelement <8 x ptr> [[TMP125]], ptr [[TMP118]], i32 2
-; CHECK-NEXT: [[TMP127:%.*]] = insertelement <8 x ptr> [[TMP126]], ptr [[TMP119]], i32 3
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <8 x ptr> [[TMP127]], ptr [[TMP120]], i32 4
-; CHECK-NEXT: [[TMP129:%.*]] = insertelement <8 x ptr> [[TMP128]], ptr [[TMP121]], i32 5
-; CHECK-NEXT: [[TMP130:%.*]] = insertelement <8 x ptr> [[TMP129]], ptr [[TMP122]], i32 6
-; CHECK-NEXT: [[TMP131:%.*]] = insertelement <8 x ptr> [[TMP130]], ptr [[TMP123]], i32 7
-; CHECK-NEXT: [[INDEX_NEXT28]] = add nuw i64 [[INDEX19]], 8
-; CHECK-NEXT: [[TMP132:%.*]] = icmp eq i64 [[INDEX_NEXT28]], [[N_VEC18]]
-; CHECK-NEXT: br i1 [[TMP132]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP133:%.*]] = ptrtoint <8 x ptr> [[TMP131]] to <8 x i64>
-; CHECK-NEXT: [[TMP134:%.*]] = extractelement <8 x i64> [[TMP133]], i64 7
-; CHECK-NEXT: [[CMP_N29:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC18]]
-; CHECK-NEXT: br i1 [[CMP_N29]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL30:%.*]] = phi i64 [ [[N_VEC18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL31:%.*]] = phi ptr [ [[TMP89]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL30]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL31]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL16]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[PTR_IV]], align 1
; CHECK-NEXT: [[VAL_EXT:%.*]] = zext i8 [[VAL]] to i32
; CHECK-NEXT: store i32 [[VAL_EXT]], ptr [[DST]], align 4
@@ -964,9 +863,9 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[PTR_INT:%.*]] = ptrtoint ptr [[PTR_IV_NEXT]] to i64
; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[IV]], [[COUNT]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[PTR_INT_LCSSA:%.*]] = phi i64 [ [[PTR_INT]], %[[LOOP]] ], [ [[TMP87]], %[[MIDDLE_BLOCK]] ], [ [[TMP134]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[PTR_INT_LCSSA:%.*]] = phi i64 [ [[PTR_INT]], %[[LOOP]] ], [ [[TMP87]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[PTR_INT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index 0c68f9343b5f8..8ead606ac995f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -237,48 +237,29 @@ define void @test_load_gep_widen_induction(ptr noalias %dst, ptr noalias %dst2)
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 6
-; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 7
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i128, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP5]], i32 0
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x ptr> [[TMP26]], ptr [[TMP6]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP20]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x ptr> [[TMP27]], ptr [[TMP8]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP23]]
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP9]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP18]], ptr [[TMP10]], i32 1
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP11]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x ptr> [[TMP22]], ptr [[TMP17]], i32 1
-; CHECK-NEXT: store ptr null, ptr [[TMP5]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP6]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP7]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP8]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP9]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP10]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP11]], align 8
-; CHECK-NEXT: store ptr null, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr i128, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i128, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT: store ptr null, ptr [[TMP39]], align 8
+; CHECK-NEXT: store ptr null, ptr [[TMP40]], align 8
+; CHECK-NEXT: store ptr null, ptr [[TMP41]], align 8
+; CHECK-NEXT: store ptr null, ptr [[TMP42]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[DST2]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 4
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 6
-; CHECK-NEXT: store <2 x ptr> [[TMP0]], ptr [[TMP12]], align 8
-; CHECK-NEXT: store <2 x ptr> [[TMP1]], ptr [[TMP13]], align 8
-; CHECK-NEXT: store <2 x ptr> [[TMP2]], ptr [[TMP14]], align 8
-; CHECK-NEXT: store <2 x ptr> [[TMP3]], ptr [[TMP15]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr ptr, ptr [[DST2]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr ptr, ptr [[DST2]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr ptr, ptr [[DST2]], i64 [[TMP21]]
+; CHECK-NEXT: store ptr [[TMP39]], ptr [[TMP12]], align 8
+; CHECK-NEXT: store ptr [[TMP40]], ptr [[TMP8]], align 8
+; CHECK-NEXT: store ptr [[TMP41]], ptr [[TMP9]], align 8
+; CHECK-NEXT: store ptr [[TMP42]], ptr [[TMP10]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -337,7 +318,7 @@ define ptr @replicating_store_in_conditional_latch(ptr %p, i32 %n) {
; CHECK-NEXT: store ptr null, ptr [[TMP15]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -501,21 +482,21 @@ define void @test_prefer_vector_addressing(ptr %start, ptr %ms, ptr noalias %src
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA12:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA11:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA11]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA11]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA11]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA17:![0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA17]]
+; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA16:![0-9]+]]
+; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA16]]
+; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA16]]
+; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA16]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -693,84 +674,44 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[SCALE]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 16
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 18
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 20
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 22
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 24
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
-; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP31:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP15]]
-; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP16]], align 1
-; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP17]], align 1
-; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP18]], align 1
-; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP19]], align 1
-; CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP20]], align 1
-; CHECK-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP22]], align 1
-; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP23]], align 1
-; CHECK-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP24]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP25]], align 1
-; CHECK-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP26]], align 1
-; CHECK-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP27]], align 1
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP44:%.*]] = load i8, ptr [[TMP28]], align 1
; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[TMP29]], align 1
; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[TMP30]], align 1
; CHECK-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP31]], align 1
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <16 x i8> poison, i8 [[TMP32]], i32 0
-; CHECK-NEXT: [[TMP49:%.*]] = insertelement <16 x i8> [[TMP48]], i8 [[TMP33]], i32 1
-; CHECK-NEXT: [[TMP50:%.*]] = insertelement <16 x i8> [[TMP49]], i8 [[TMP34]], i32 2
-; CHECK-NEXT: [[TMP51:%.*]] = insertelement <16 x i8> [[TMP50]], i8 [[TMP35]], i32 3
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP36]], i32 4
-; CHECK-NEXT: [[TMP53:%.*]] = insertelement <16 x i8> [[TMP52]], i8 [[TMP37]], i32 5
-; CHECK-NEXT: [[TMP54:%.*]] = insertelement <16 x i8> [[TMP53]], i8 [[TMP38]], i32 6
-; CHECK-NEXT: [[TMP55:%.*]] = insertelement <16 x i8> [[TMP54]], i8 [[TMP39]], i32 7
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <16 x i8> [[TMP55]], i8 [[TMP40]], i32 8
-; CHECK-NEXT: [[TMP57:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP41]], i32 9
-; CHECK-NEXT: [[TMP58:%.*]] = insertelement <16 x i8> [[TMP57]], i8 [[TMP42]], i32 10
-; CHECK-NEXT: [[TMP59:%.*]] = insertelement <16 x i8> [[TMP58]], i8 [[TMP43]], i32 11
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <16 x i8> [[TMP59]], i8 [[TMP44]], i32 12
-; CHECK-NEXT: [[TMP61:%.*]] = insertelement <16 x i8> [[TMP60]], i8 [[TMP45]], i32 13
-; CHECK-NEXT: [[TMP62:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP46]], i32 14
-; CHECK-NEXT: [[TMP63:%.*]] = insertelement <16 x i8> [[TMP62]], i8 [[TMP47]], i32 15
-; CHECK-NEXT: [[TMP64:%.*]] = sext <16 x i8> [[TMP63]] to <16 x i32>
-; CHECK-NEXT: [[TMP65:%.*]] = mul <16 x i32> [[BROADCAST_SPLAT]], [[TMP64]]
-; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP12:%.*]] = sext i8 [[TMP44]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = sext i8 [[TMP45]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = sext i8 [[TMP46]] to i32
+; CHECK-NEXT: [[TMP15:%.*]] = sext i8 [[TMP47]] to i32
+; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[SCALE]], [[TMP12]]
+; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[SCALE]], [[TMP13]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[SCALE]], [[TMP14]]
+; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[SCALE]], [[TMP15]]
+; CHECK-NEXT: [[TMP20]] = or i32 [[TMP16]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP21]] = or i32 [[TMP17]], [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP22]] = or i32 [[TMP18]], [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP23]] = or i32 [[TMP19]], [[VEC_PHI3]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = or i32 [[TMP21]], [[TMP20]]
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = or i32 [[TMP22]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = or i32 [[TMP23]], [[BIN_RDX4]]
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index 9a60827951036..fb07f2f87189a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -658,86 +658,31 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[SCALE]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 16
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 18
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 20
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 22
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 24
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
-; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP31:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP15]]
-; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP16]], align 1
-; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP17]], align 1
-; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP18]], align 1
-; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP19]], align 1
-; CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP20]], align 1
-; CHECK-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP22]], align 1
-; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP23]], align 1
-; CHECK-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP24]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP25]], align 1
-; CHECK-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP26]], align 1
-; CHECK-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP27]], align 1
-; CHECK-NEXT: [[TMP44:%.*]] = load i8, ptr [[TMP28]], align 1
-; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[TMP29]], align 1
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[TMP30]], align 1
; CHECK-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP31]], align 1
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <16 x i8> poison, i8 [[TMP32]], i32 0
-; CHECK-NEXT: [[TMP49:%.*]] = insertelement <16 x i8> [[TMP48]], i8 [[TMP33]], i32 1
-; CHECK-NEXT: [[TMP50:%.*]] = insertelement <16 x i8> [[TMP49]], i8 [[TMP34]], i32 2
-; CHECK-NEXT: [[TMP51:%.*]] = insertelement <16 x i8> [[TMP50]], i8 [[TMP35]], i32 3
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP36]], i32 4
-; CHECK-NEXT: [[TMP53:%.*]] = insertelement <16 x i8> [[TMP52]], i8 [[TMP37]], i32 5
-; CHECK-NEXT: [[TMP54:%.*]] = insertelement <16 x i8> [[TMP53]], i8 [[TMP38]], i32 6
-; CHECK-NEXT: [[TMP55:%.*]] = insertelement <16 x i8> [[TMP54]], i8 [[TMP39]], i32 7
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <16 x i8> [[TMP55]], i8 [[TMP40]], i32 8
-; CHECK-NEXT: [[TMP57:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP41]], i32 9
-; CHECK-NEXT: [[TMP58:%.*]] = insertelement <16 x i8> [[TMP57]], i8 [[TMP42]], i32 10
-; CHECK-NEXT: [[TMP59:%.*]] = insertelement <16 x i8> [[TMP58]], i8 [[TMP43]], i32 11
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <16 x i8> [[TMP59]], i8 [[TMP44]], i32 12
-; CHECK-NEXT: [[TMP61:%.*]] = insertelement <16 x i8> [[TMP60]], i8 [[TMP45]], i32 13
-; CHECK-NEXT: [[TMP62:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP46]], i32 14
-; CHECK-NEXT: [[TMP63:%.*]] = insertelement <16 x i8> [[TMP62]], i8 [[TMP47]], i32 15
-; CHECK-NEXT: [[TMP64:%.*]] = sext <16 x i8> [[TMP63]] to <16 x i32>
-; CHECK-NEXT: [[TMP65:%.*]] = mul <16 x i32> [[BROADCAST_SPLAT]], [[TMP64]]
-; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: [[TMP6:%.*]] = sext i8 [[TMP46]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = sext i8 [[TMP47]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = mul i32 [[SCALE]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = mul i32 [[SCALE]], [[TMP7]]
+; CHECK-NEXT: [[TMP10]] = or i32 [[TMP8]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP11]] = or i32 [[TMP9]], [[VEC_PHI1]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 50
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = or i32 [[TMP11]], [[TMP10]]
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[BIN_RDX]]
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
index aa448cdd6c689..64c474a3584b5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
@@ -5,8 +5,8 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
define i32 @scalar_steps_all_lanes(ptr %start, ptr %end) {
; CHECK-LABEL: LV: Checking a loop in 'scalar_steps_all_lanes'
-; CHECK: Cost of 0 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
-; CHECK: Cost of 0 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 1 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 3 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
index 326e7ddcd0647..49d89e2a11332 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
@@ -172,10 +172,10 @@ exit:
define void @spills_profitable(ptr %in1, ptr %in2, ptr %out, i32 %n, i32 %m) {
; CHECK-LABEL: LV: Checking a loop in 'spills_profitable'
; CHECK: LV: Scalar loop costs: 54
-; CHECK-NOPRESSURE: Cost for VF 2: 1535 (Estimated cost per lane: 767.
+; CHECK-NOPRESSURE: Cost for VF 2: 1540 (Estimated cost per lane: 770.
; CHECK-NOPRESSURE: Cost for VF 4: 43 (Estimated cost per lane: 10.
; CHECK-PRESSURE: LV(REG): Cost of 4 from 2 spills of Generic::ScalarRC
-; CHECK-PRESSURE-NEXT: Cost for VF 2: 1539 (Estimated cost per lane: 769.
+; CHECK-PRESSURE-NEXT: Cost for VF 2: 1544 (Estimated cost per lane: 772.
; CHECK-PRESSURE: LV(REG): Cost of 6 from 3 spills of Generic::VectorRC
; CHECK-PRESSURE-NEXT: Cost for VF 4: 49 (Estimated cost per lane: 12.
; CHECK: LV: Selecting VF: 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index 64d412666c74f..03507862fa425 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -581,18 +581,23 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
; FIXEDLEN-NEXT: [[ENTRY:.*:]]
; FIXEDLEN-NEXT: br label %[[VECTOR_PH:.*]]
; FIXEDLEN: [[VECTOR_PH]]:
-; FIXEDLEN-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
-; FIXEDLEN-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; FIXEDLEN-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[B]], i64 0
+; FIXEDLEN-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; FIXEDLEN-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[V]], i64 0
+; FIXEDLEN-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
; FIXEDLEN-NEXT: br label %[[VECTOR_BODY:.*]]
; FIXEDLEN: [[VECTOR_BODY]]:
; FIXEDLEN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXEDLEN-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 7
-; FIXEDLEN-NEXT: store i64 [[TMP4]], ptr [[B]], align 8
+; FIXEDLEN-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXEDLEN-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; FIXEDLEN-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VEC_IND]], <2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true))
+; FIXEDLEN-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[STEP_ADD]], <2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true))
; FIXEDLEN-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; FIXEDLEN-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 4
-; FIXEDLEN-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 8
-; FIXEDLEN-NEXT: store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8
-; FIXEDLEN-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXEDLEN-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i64 2
+; FIXEDLEN-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP5]], align 8
+; FIXEDLEN-NEXT: store <2 x i64> [[BROADCAST_SPLAT2]], ptr [[TMP1]], align 8
+; FIXEDLEN-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; FIXEDLEN-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)
; FIXEDLEN-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXEDLEN-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; FIXEDLEN: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
index 6a0a3410a9585..0905b91ad7023 100644
--- a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
@@ -113,11 +113,11 @@ define hidden void @two_ints_vary_op(ptr noalias nocapture noundef writeonly %0,
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 61 (Estimated cost per lane: 30.
+; CHECK: Cost for VF 2: 62 (Estimated cost per lane: 31.
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 12 for VF 4: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 4: 115 (Estimated cost per lane: 28.
+; CHECK: Cost for VF 4: 118 (Estimated cost per lane: 29.
; CHECK: LV: Selecting VF: 1.
define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
%5 = icmp eq i32 %3, 0
@@ -159,15 +159,15 @@ define hidden void @three_ints(ptr noalias nocapture noundef writeonly %0, ptr n
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 61 (Estimated cost per lane: 30.
+; CHECK: Cost for VF 2: 62 (Estimated cost per lane: 31.
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 12 for VF 4: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 4: 115 (Estimated cost per lane: 28.
+; CHECK: Cost for VF 4: 118 (Estimated cost per lane: 29.
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 24 for VF 8: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 8: 223 (Estimated cost per lane: 27.
+; CHECK: Cost for VF 8: 230 (Estimated cost per lane: 28.
; CHECK: LV: Selecting VF: 1.
define hidden void @three_shorts(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
%5 = icmp eq i32 %3, 0
@@ -443,7 +443,7 @@ define hidden void @four_shorts_interleave_op(ptr noalias nocapture noundef writ
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 99 (Estimated cost per lane: 49.
+; CHECK: Cost for VF 2: 100 (Estimated cost per lane: 50.
; CHECK: Cost of 42 for VF 4: INTERLEAVE-GROUP with factor 5, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -538,7 +538,7 @@ define hidden void @five_shorts(ptr noalias nocapture noundef writeonly %0, ptr
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26.
+; CHECK: Cost for VF 2: 53 (Estimated cost per lane: 26.
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -594,7 +594,7 @@ define hidden void @two_bytes_same_op(ptr noalias nocapture noundef writeonly %0
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 47 (Estimated cost per lane: 23.
+; CHECK: Cost for VF 2: 48 (Estimated cost per lane: 24.
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -650,19 +650,19 @@ define hidden void @two_bytes_vary_op(ptr noalias nocapture noundef writeonly %0
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 61 (Estimated cost per lane: 30.
+; CHECK: Cost for VF 2: 62 (Estimated cost per lane: 31.
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 12 for VF 4: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 4: 115 (Estimated cost per lane: 28.
+; CHECK: Cost for VF 4: 118 (Estimated cost per lane: 29.
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 24 for VF 8: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 8: 223 (Estimated cost per lane: 27.
+; CHECK: Cost for VF 8: 230 (Estimated cost per lane: 28.
; CHECK: Cost of 48 for VF 16: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 48 for VF 16: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 48 for VF 16: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 16: 439 (Estimated cost per lane: 27.
+; CHECK: Cost for VF 16: 454 (Estimated cost per lane: 28.
; CHECK: LV: Selecting VF: 1.
define hidden void @three_bytes_same_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
%5 = icmp eq i32 %3, 0
@@ -704,19 +704,19 @@ define hidden void @three_bytes_same_op(ptr noalias nocapture noundef writeonly
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 61 (Estimated cost per lane: 30.
+; CHECK: Cost for VF 2: 62 (Estimated cost per lane: 31.
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 12 for VF 4: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 4: 115 (Estimated cost per lane: 28.
+; CHECK: Cost for VF 4: 118 (Estimated cost per lane: 29.
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 24 for VF 8: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 24 for VF 8: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 8: 223 (Estimated cost per lane: 27.
+; CHECK: Cost for VF 8: 230 (Estimated cost per lane: 28.
; CHECK: Cost of 48 for VF 16: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 48 for VF 16: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 48 for VF 16: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 16: 439 (Estimated cost per lane: 27.
+; CHECK: Cost for VF 16: 454 (Estimated cost per lane: 28.
; CHECK: LV: Selecting VF: 1.
define hidden void @three_bytes_interleave_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
%5 = icmp eq i32 %3, 0
@@ -758,7 +758,7 @@ define hidden void @three_bytes_interleave_op(ptr noalias nocapture noundef writ
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 80 (Estimated cost per lane: 40.
+; CHECK: Cost for VF 2: 81 (Estimated cost per lane: 40.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -840,7 +840,7 @@ define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 90 (Estimated cost per lane: 45.
+; CHECK: Cost for VF 2: 91 (Estimated cost per lane: 45.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -923,7 +923,7 @@ define hidden void @four_bytes_split_op(ptr noalias nocapture noundef writeonly
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%10> = load ir<%9>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%13>, ir<%14>
-; CHECK: Cost for VF 2: 80 (Estimated cost per lane: 40.
+; CHECK: Cost for VF 2: 81 (Estimated cost per lane: 40.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%16> = load from index 1
@@ -1463,7 +1463,7 @@ define hidden void @eight_bytes_interleave_op(ptr noalias nocapture noundef writ
; CHECK-NEXT: store ir<%28> to index 1
; CHECK-NEXT: store ir<%38> to index 2
; CHECK-NEXT: store ir<%48> to index 3
-; CHECK: Cost for VF 2: 88 (Estimated cost per lane: 44.
+; CHECK: Cost for VF 2: 89 (Estimated cost per lane: 44.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%20> = load from index 1
@@ -1550,7 +1550,7 @@ define hidden void @four_bytes_into_four_ints_same_op(ptr noalias nocapture noun
; CHECK-NEXT: store ir<%23> to index 1
; CHECK-NEXT: store ir<%31> to index 2
; CHECK-NEXT: store ir<%38> to index 3
-; CHECK: Cost for VF 2: 71 (Estimated cost per lane: 35.
+; CHECK: Cost for VF 2: 72 (Estimated cost per lane: 36.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%9>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%18> = load from index 1
@@ -1668,7 +1668,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%17> = load ir<%16> (!alias.scope {{.*}})
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19> (!alias.scope {{.*}})
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%48>, ir<%49>
-; CHECK: Cost for VF 2: 80 (Estimated cost per lane: 40.
+; CHECK: Cost for VF 2: 82 (Estimated cost per lane: 41.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK-NEXT: ir<%14> = load from index 0
; CHECK-NEXT: ir<%32> = load from index 1
@@ -1768,7 +1768,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%13> = load ir<%12> (!alias.scope {{.*}})
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%18>, vp<%next.gep>.1 (!alias.scope {{.*}}, !noalias {{.*}})
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%28>, ir<%29>
-; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26.
+; CHECK: Cost for VF 2: 54 (Estimated cost per lane: 27.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK-NEXT: ir<%10> = load from index 0
; CHECK-NEXT: ir<%20> = load from index 1
@@ -1949,7 +1949,7 @@ for.body:
; CHECK: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: store ir<%mul> to index 0
; CHECK-NEXT: store ir<%mul11> to index 1
-; CHECK: Cost for VF 2: 51 (Estimated cost per lane: 25.
+; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26.
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2002,7 +2002,7 @@ for.body:
; CHECK: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: store ir<%add> to index 0
; CHECK-NEXT: store ir<%sub> to index 1
-; CHECK: Cost for VF 2: 51 (Estimated cost per lane: 25.
+; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26.
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2057,7 +2057,7 @@ for.body:
; CHECK-NEXT: ir<%3> = load from index 1
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv>, ir<%arrayidx3>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv9>, ir<%y11>
-; CHECK: Cost for VF 2: 46 (Estimated cost per lane: 23.
+; CHECK: Cost for VF 2: 47 (Estimated cost per lane: 23.
; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2110,7 +2110,7 @@ for.body:
; CHECK-NEXT: ir<%3> = load from index 1
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv8>
-; CHECK: Cost for VF 2: 46 (Estimated cost per lane: 23.
+; CHECK: Cost for VF 2: 47 (Estimated cost per lane: 23.
; CHECK: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 2
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2524,7 +2524,7 @@ for.body:
; CHECK-NEXT: store ir<%mul11> to index 1
; CHECK-NEXT: store ir<%mul19> to index 2
; CHECK-NEXT: store ir<%mul27> to index 3
-; CHECK: Cost for VF 2: 98 (Estimated cost per lane: 49.
+; CHECK: Cost for VF 2: 99 (Estimated cost per lane: 49.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2603,7 +2603,7 @@ for.body:
; CHECK-NEXT: store ir<%add> to index 1
; CHECK-NEXT: store ir<%div> to index 2
; CHECK-NEXT: store ir<%sub> to index 3
-; CHECK: Cost for VF 2: 98 (Estimated cost per lane: 49.
+; CHECK: Cost for VF 2: 99 (Estimated cost per lane: 49.
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2687,7 +2687,7 @@ for.body:
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv>, ir<%arrayidx3>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv9>, ir<%y11>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv16>, ir<%z18>
-; CHECK: Cost for VF 2: 88 (Estimated cost per lane: 44.
+; CHECK: Cost for VF 2: 89 (Estimated cost per lane: 44.
; CHECK: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx>
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
@@ -2763,7 +2763,7 @@ for.body:
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv>, ir<%arrayidx3>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv8>, ir<%y10>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%conv14>, ir<%z16>
-; CHECK: Cost for VF 2: 88 (Estimated cost per lane: 44.
+; CHECK: Cost for VF 2: 89 (Estimated cost per lane: 44.
; CHECK: Cost of 36 for VF 4: INTERLEAVE-GROUP with factor 4, ir<%arrayidx>
; CHECK-NEXT: ir<%0> = load from index 0
; CHECK-NEXT: ir<%2> = load from index 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 47215a17934be..4434213a842fc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -79,19 +79,19 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], 32
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 32
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 5
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP232:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP233:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP234:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP235:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP120:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP121:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP122:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP123:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[INDEX]], 5
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 32
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], 64
@@ -108,22 +108,6 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP4]], 416
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP4]], 448
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], 480
-; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[TMP4]], 512
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP4]], 544
-; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP4]], 576
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP4]], 608
-; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP4]], 640
-; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP4]], 672
-; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP4]], 704
-; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP4]], 736
-; CHECK-NEXT: [[TMP28:%.*]] = add i64 [[TMP4]], 768
-; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP4]], 800
-; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[TMP4]], 832
-; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP4]], 864
-; CHECK-NEXT: [[TMP32:%.*]] = add i64 [[TMP4]], 896
-; CHECK-NEXT: [[TMP33:%.*]] = add i64 [[TMP4]], 928
-; CHECK-NEXT: [[TMP34:%.*]] = add i64 [[TMP4]], 960
-; CHECK-NEXT: [[TMP35:%.*]] = add i64 [[TMP4]], 992
; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]]
@@ -140,86 +124,38 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP17]]
; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]]
-; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]]
-; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP27]]
-; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP28]]
-; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP29]]
-; CHECK-NEXT: [[TMP62:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP30]]
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP32]]
-; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP33]]
-; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP35]]
; CHECK-NEXT: [[TMP68:%.*]] = load float, ptr [[TMP36]], align 4
; CHECK-NEXT: [[TMP69:%.*]] = load float, ptr [[TMP37]], align 4
; CHECK-NEXT: [[TMP70:%.*]] = load float, ptr [[TMP38]], align 4
; CHECK-NEXT: [[TMP71:%.*]] = load float, ptr [[TMP39]], align 4
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x float> poison, float [[TMP68]], i32 0
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <4 x float> [[TMP52]], float [[TMP69]], i32 1
+; CHECK-NEXT: [[TMP54:%.*]] = insertelement <4 x float> [[TMP53]], float [[TMP70]], i32 2
+; CHECK-NEXT: [[TMP55:%.*]] = insertelement <4 x float> [[TMP54]], float [[TMP71]], i32 3
; CHECK-NEXT: [[TMP72:%.*]] = load float, ptr [[TMP40]], align 4
; CHECK-NEXT: [[TMP73:%.*]] = load float, ptr [[TMP41]], align 4
; CHECK-NEXT: [[TMP74:%.*]] = load float, ptr [[TMP42]], align 4
; CHECK-NEXT: [[TMP75:%.*]] = load float, ptr [[TMP43]], align 4
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <8 x float> poison, float [[TMP68]], i32 0
-; CHECK-NEXT: [[TMP77:%.*]] = insertelement <8 x float> [[TMP76]], float [[TMP69]], i32 1
-; CHECK-NEXT: [[TMP78:%.*]] = insertelement <8 x float> [[TMP77]], float [[TMP70]], i32 2
-; CHECK-NEXT: [[TMP79:%.*]] = insertelement <8 x float> [[TMP78]], float [[TMP71]], i32 3
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <8 x float> [[TMP79]], float [[TMP72]], i32 4
-; CHECK-NEXT: [[TMP81:%.*]] = insertelement <8 x float> [[TMP80]], float [[TMP73]], i32 5
-; CHECK-NEXT: [[TMP82:%.*]] = insertelement <8 x float> [[TMP81]], float [[TMP74]], i32 6
-; CHECK-NEXT: [[TMP83:%.*]] = insertelement <8 x float> [[TMP82]], float [[TMP75]], i32 7
+; CHECK-NEXT: [[TMP60:%.*]] = insertelement <4 x float> poison, float [[TMP72]], i32 0
+; CHECK-NEXT: [[TMP61:%.*]] = insertelement <4 x float> [[TMP60]], float [[TMP73]], i32 1
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <4 x float> [[TMP61]], float [[TMP74]], i32 2
+; CHECK-NEXT: [[TMP63:%.*]] = insertelement <4 x float> [[TMP62]], float [[TMP75]], i32 3
; CHECK-NEXT: [[TMP84:%.*]] = load float, ptr [[TMP44]], align 4
; CHECK-NEXT: [[TMP85:%.*]] = load float, ptr [[TMP45]], align 4
; CHECK-NEXT: [[TMP86:%.*]] = load float, ptr [[TMP46]], align 4
; CHECK-NEXT: [[TMP87:%.*]] = load float, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP56:%.*]] = insertelement <4 x float> poison, float [[TMP84]], i32 0
+; CHECK-NEXT: [[TMP57:%.*]] = insertelement <4 x float> [[TMP56]], float [[TMP85]], i32 1
+; CHECK-NEXT: [[TMP58:%.*]] = insertelement <4 x float> [[TMP57]], float [[TMP86]], i32 2
+; CHECK-NEXT: [[TMP59:%.*]] = insertelement <4 x float> [[TMP58]], float [[TMP87]], i32 3
; CHECK-NEXT: [[TMP88:%.*]] = load float, ptr [[TMP48]], align 4
; CHECK-NEXT: [[TMP89:%.*]] = load float, ptr [[TMP49]], align 4
; CHECK-NEXT: [[TMP90:%.*]] = load float, ptr [[TMP50]], align 4
; CHECK-NEXT: [[TMP91:%.*]] = load float, ptr [[TMP51]], align 4
-; CHECK-NEXT: [[TMP92:%.*]] = insertelement <8 x float> poison, float [[TMP84]], i32 0
-; CHECK-NEXT: [[TMP93:%.*]] = insertelement <8 x float> [[TMP92]], float [[TMP85]], i32 1
-; CHECK-NEXT: [[TMP94:%.*]] = insertelement <8 x float> [[TMP93]], float [[TMP86]], i32 2
-; CHECK-NEXT: [[TMP95:%.*]] = insertelement <8 x float> [[TMP94]], float [[TMP87]], i32 3
-; CHECK-NEXT: [[TMP96:%.*]] = insertelement <8 x float> [[TMP95]], float [[TMP88]], i32 4
-; CHECK-NEXT: [[TMP97:%.*]] = insertelement <8 x float> [[TMP96]], float [[TMP89]], i32 5
-; CHECK-NEXT: [[TMP98:%.*]] = insertelement <8 x float> [[TMP97]], float [[TMP90]], i32 6
-; CHECK-NEXT: [[TMP99:%.*]] = insertelement <8 x float> [[TMP98]], float [[TMP91]], i32 7
-; CHECK-NEXT: [[TMP100:%.*]] = load float, ptr [[TMP52]], align 4
-; CHECK-NEXT: [[TMP101:%.*]] = load float, ptr [[TMP53]], align 4
-; CHECK-NEXT: [[TMP102:%.*]] = load float, ptr [[TMP54]], align 4
-; CHECK-NEXT: [[TMP103:%.*]] = load float, ptr [[TMP55]], align 4
-; CHECK-NEXT: [[TMP104:%.*]] = load float, ptr [[TMP56]], align 4
-; CHECK-NEXT: [[TMP105:%.*]] = load float, ptr [[TMP57]], align 4
-; CHECK-NEXT: [[TMP106:%.*]] = load float, ptr [[TMP58]], align 4
-; CHECK-NEXT: [[TMP107:%.*]] = load float, ptr [[TMP59]], align 4
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <8 x float> poison, float [[TMP100]], i32 0
-; CHECK-NEXT: [[TMP109:%.*]] = insertelement <8 x float> [[TMP108]], float [[TMP101]], i32 1
-; CHECK-NEXT: [[TMP110:%.*]] = insertelement <8 x float> [[TMP109]], float [[TMP102]], i32 2
-; CHECK-NEXT: [[TMP111:%.*]] = insertelement <8 x float> [[TMP110]], float [[TMP103]], i32 3
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <8 x float> [[TMP111]], float [[TMP104]], i32 4
-; CHECK-NEXT: [[TMP113:%.*]] = insertelement <8 x float> [[TMP112]], float [[TMP105]], i32 5
-; CHECK-NEXT: [[TMP114:%.*]] = insertelement <8 x float> [[TMP113]], float [[TMP106]], i32 6
-; CHECK-NEXT: [[TMP115:%.*]] = insertelement <8 x float> [[TMP114]], float [[TMP107]], i32 7
-; CHECK-NEXT: [[TMP116:%.*]] = load float, ptr [[TMP60]], align 4
-; CHECK-NEXT: [[TMP117:%.*]] = load float, ptr [[TMP61]], align 4
-; CHECK-NEXT: [[TMP118:%.*]] = load float, ptr [[TMP62]], align 4
-; CHECK-NEXT: [[TMP119:%.*]] = load float, ptr [[TMP63]], align 4
-; CHECK-NEXT: [[TMP120:%.*]] = load float, ptr [[TMP64]], align 4
-; CHECK-NEXT: [[TMP121:%.*]] = load float, ptr [[TMP65]], align 4
-; CHECK-NEXT: [[TMP122:%.*]] = load float, ptr [[TMP66]], align 4
-; CHECK-NEXT: [[TMP123:%.*]] = load float, ptr [[TMP67]], align 4
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <8 x float> poison, float [[TMP116]], i32 0
-; CHECK-NEXT: [[TMP125:%.*]] = insertelement <8 x float> [[TMP124]], float [[TMP117]], i32 1
-; CHECK-NEXT: [[TMP126:%.*]] = insertelement <8 x float> [[TMP125]], float [[TMP118]], i32 2
-; CHECK-NEXT: [[TMP127:%.*]] = insertelement <8 x float> [[TMP126]], float [[TMP119]], i32 3
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <8 x float> [[TMP127]], float [[TMP120]], i32 4
-; CHECK-NEXT: [[TMP129:%.*]] = insertelement <8 x float> [[TMP128]], float [[TMP121]], i32 5
-; CHECK-NEXT: [[TMP130:%.*]] = insertelement <8 x float> [[TMP129]], float [[TMP122]], i32 6
-; CHECK-NEXT: [[TMP131:%.*]] = insertelement <8 x float> [[TMP130]], float [[TMP123]], i32 7
+; CHECK-NEXT: [[TMP64:%.*]] = insertelement <4 x float> poison, float [[TMP88]], i32 0
+; CHECK-NEXT: [[TMP65:%.*]] = insertelement <4 x float> [[TMP64]], float [[TMP89]], i32 1
+; CHECK-NEXT: [[TMP66:%.*]] = insertelement <4 x float> [[TMP65]], float [[TMP90]], i32 2
+; CHECK-NEXT: [[TMP67:%.*]] = insertelement <4 x float> [[TMP66]], float [[TMP91]], i32 3
; CHECK-NEXT: [[TMP132:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP133:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP134:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP6]]
@@ -236,102 +172,54 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
; CHECK-NEXT: [[TMP145:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
; CHECK-NEXT: [[TMP146:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP147:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP148:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP20]]
-; CHECK-NEXT: [[TMP149:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP150:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP151:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP23]]
-; CHECK-NEXT: [[TMP152:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP153:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP154:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP155:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP27]]
-; CHECK-NEXT: [[TMP156:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP28]]
-; CHECK-NEXT: [[TMP157:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP29]]
-; CHECK-NEXT: [[TMP158:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP30]]
-; CHECK-NEXT: [[TMP159:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP160:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP32]]
-; CHECK-NEXT: [[TMP161:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP33]]
-; CHECK-NEXT: [[TMP162:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP163:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP35]]
; CHECK-NEXT: [[TMP164:%.*]] = load float, ptr [[TMP132]], align 4
; CHECK-NEXT: [[TMP165:%.*]] = load float, ptr [[TMP133]], align 4
; CHECK-NEXT: [[TMP166:%.*]] = load float, ptr [[TMP134]], align 4
; CHECK-NEXT: [[TMP167:%.*]] = load float, ptr [[TMP135]], align 4
+; CHECK-NEXT: [[TMP92:%.*]] = insertelement <4 x float> poison, float [[TMP164]], i32 0
+; CHECK-NEXT: [[TMP93:%.*]] = insertelement <4 x float> [[TMP92]], float [[TMP165]], i32 1
+; CHECK-NEXT: [[TMP94:%.*]] = insertelement <4 x float> [[TMP93]], float [[TMP166]], i32 2
+; CHECK-NEXT: [[TMP95:%.*]] = insertelement <4 x float> [[TMP94]], float [[TMP167]], i32 3
; CHECK-NEXT: [[TMP168:%.*]] = load float, ptr [[TMP136]], align 4
; CHECK-NEXT: [[TMP169:%.*]] = load float, ptr [[TMP137]], align 4
; CHECK-NEXT: [[TMP170:%.*]] = load float, ptr [[TMP138]], align 4
; CHECK-NEXT: [[TMP171:%.*]] = load float, ptr [[TMP139]], align 4
-; CHECK-NEXT: [[TMP172:%.*]] = insertelement <8 x float> poison, float [[TMP164]], i32 0
-; CHECK-NEXT: [[TMP173:%.*]] = insertelement <8 x float> [[TMP172]], float [[TMP165]], i32 1
-; CHECK-NEXT: [[TMP174:%.*]] = insertelement <8 x float> [[TMP173]], float [[TMP166]], i32 2
-; CHECK-NEXT: [[TMP175:%.*]] = insertelement <8 x float> [[TMP174]], float [[TMP167]], i32 3
-; CHECK-NEXT: [[TMP176:%.*]] = insertelement <8 x float> [[TMP175]], float [[TMP168]], i32 4
-; CHECK-NEXT: [[TMP177:%.*]] = insertelement <8 x float> [[TMP176]], float [[TMP169]], i32 5
-; CHECK-NEXT: [[TMP178:%.*]] = insertelement <8 x float> [[TMP177]], float [[TMP170]], i32 6
-; CHECK-NEXT: [[TMP179:%.*]] = insertelement <8 x float> [[TMP178]], float [[TMP171]], i32 7
+; CHECK-NEXT: [[TMP96:%.*]] = insertelement <4 x float> poison, float [[TMP168]], i32 0
+; CHECK-NEXT: [[TMP97:%.*]] = insertelement <4 x float> [[TMP96]], float [[TMP169]], i32 1
+; CHECK-NEXT: [[TMP98:%.*]] = insertelement <4 x float> [[TMP97]], float [[TMP170]], i32 2
+; CHECK-NEXT: [[TMP99:%.*]] = insertelement <4 x float> [[TMP98]], float [[TMP171]], i32 3
; CHECK-NEXT: [[TMP180:%.*]] = load float, ptr [[TMP140]], align 4
; CHECK-NEXT: [[TMP181:%.*]] = load float, ptr [[TMP141]], align 4
; CHECK-NEXT: [[TMP182:%.*]] = load float, ptr [[TMP142]], align 4
; CHECK-NEXT: [[TMP183:%.*]] = load float, ptr [[TMP143]], align 4
+; CHECK-NEXT: [[TMP104:%.*]] = insertelement <4 x float> poison, float [[TMP180]], i32 0
+; CHECK-NEXT: [[TMP105:%.*]] = insertelement <4 x float> [[TMP104]], float [[TMP181]], i32 1
+; CHECK-NEXT: [[TMP106:%.*]] = insertelement <4 x float> [[TMP105]], float [[TMP182]], i32 2
+; CHECK-NEXT: [[TMP107:%.*]] = insertelement <4 x float> [[TMP106]], float [[TMP183]], i32 3
; CHECK-NEXT: [[TMP184:%.*]] = load float, ptr [[TMP144]], align 4
; CHECK-NEXT: [[TMP185:%.*]] = load float, ptr [[TMP145]], align 4
; CHECK-NEXT: [[TMP186:%.*]] = load float, ptr [[TMP146]], align 4
; CHECK-NEXT: [[TMP187:%.*]] = load float, ptr [[TMP147]], align 4
-; CHECK-NEXT: [[TMP188:%.*]] = insertelement <8 x float> poison, float [[TMP180]], i32 0
-; CHECK-NEXT: [[TMP189:%.*]] = insertelement <8 x float> [[TMP188]], float [[TMP181]], i32 1
-; CHECK-NEXT: [[TMP190:%.*]] = insertelement <8 x float> [[TMP189]], float [[TMP182]], i32 2
-; CHECK-NEXT: [[TMP191:%.*]] = insertelement <8 x float> [[TMP190]], float [[TMP183]], i32 3
-; CHECK-NEXT: [[TMP192:%.*]] = insertelement <8 x float> [[TMP191]], float [[TMP184]], i32 4
-; CHECK-NEXT: [[TMP193:%.*]] = insertelement <8 x float> [[TMP192]], float [[TMP185]], i32 5
-; CHECK-NEXT: [[TMP194:%.*]] = insertelement <8 x float> [[TMP193]], float [[TMP186]], i32 6
-; CHECK-NEXT: [[TMP195:%.*]] = insertelement <8 x float> [[TMP194]], float [[TMP187]], i32 7
-; CHECK-NEXT: [[TMP196:%.*]] = load float, ptr [[TMP148]], align 4
-; CHECK-NEXT: [[TMP197:%.*]] = load float, ptr [[TMP149]], align 4
-; CHECK-NEXT: [[TMP198:%.*]] = load float, ptr [[TMP150]], align 4
-; CHECK-NEXT: [[TMP199:%.*]] = load float, ptr [[TMP151]], align 4
-; CHECK-NEXT: [[TMP200:%.*]] = load float, ptr [[TMP152]], align 4
-; CHECK-NEXT: [[TMP201:%.*]] = load float, ptr [[TMP153]], align 4
-; CHECK-NEXT: [[TMP202:%.*]] = load float, ptr [[TMP154]], align 4
-; CHECK-NEXT: [[TMP203:%.*]] = load float, ptr [[TMP155]], align 4
-; CHECK-NEXT: [[TMP204:%.*]] = insertelement <8 x float> poison, float [[TMP196]], i32 0
-; CHECK-NEXT: [[TMP205:%.*]] = insertelement <8 x float> [[TMP204]], float [[TMP197]], i32 1
-; CHECK-NEXT: [[TMP206:%.*]] = insertelement <8 x float> [[TMP205]], float [[TMP198]], i32 2
-; CHECK-NEXT: [[TMP207:%.*]] = insertelement <8 x float> [[TMP206]], float [[TMP199]], i32 3
-; CHECK-NEXT: [[TMP208:%.*]] = insertelement <8 x float> [[TMP207]], float [[TMP200]], i32 4
-; CHECK-NEXT: [[TMP209:%.*]] = insertelement <8 x float> [[TMP208]], float [[TMP201]], i32 5
-; CHECK-NEXT: [[TMP210:%.*]] = insertelement <8 x float> [[TMP209]], float [[TMP202]], i32 6
-; CHECK-NEXT: [[TMP211:%.*]] = insertelement <8 x float> [[TMP210]], float [[TMP203]], i32 7
-; CHECK-NEXT: [[TMP212:%.*]] = load float, ptr [[TMP156]], align 4
-; CHECK-NEXT: [[TMP213:%.*]] = load float, ptr [[TMP157]], align 4
-; CHECK-NEXT: [[TMP214:%.*]] = load float, ptr [[TMP158]], align 4
-; CHECK-NEXT: [[TMP215:%.*]] = load float, ptr [[TMP159]], align 4
-; CHECK-NEXT: [[TMP216:%.*]] = load float, ptr [[TMP160]], align 4
-; CHECK-NEXT: [[TMP217:%.*]] = load float, ptr [[TMP161]], align 4
-; CHECK-NEXT: [[TMP218:%.*]] = load float, ptr [[TMP162]], align 4
-; CHECK-NEXT: [[TMP219:%.*]] = load float, ptr [[TMP163]], align 4
-; CHECK-NEXT: [[TMP220:%.*]] = insertelement <8 x float> poison, float [[TMP212]], i32 0
-; CHECK-NEXT: [[TMP221:%.*]] = insertelement <8 x float> [[TMP220]], float [[TMP213]], i32 1
-; CHECK-NEXT: [[TMP222:%.*]] = insertelement <8 x float> [[TMP221]], float [[TMP214]], i32 2
-; CHECK-NEXT: [[TMP223:%.*]] = insertelement <8 x float> [[TMP222]], float [[TMP215]], i32 3
-; CHECK-NEXT: [[TMP224:%.*]] = insertelement <8 x float> [[TMP223]], float [[TMP216]], i32 4
-; CHECK-NEXT: [[TMP225:%.*]] = insertelement <8 x float> [[TMP224]], float [[TMP217]], i32 5
-; CHECK-NEXT: [[TMP226:%.*]] = insertelement <8 x float> [[TMP225]], float [[TMP218]], i32 6
-; CHECK-NEXT: [[TMP227:%.*]] = insertelement <8 x float> [[TMP226]], float [[TMP219]], i32 7
-; CHECK-NEXT: [[TMP228:%.*]] = fadd fast <8 x float> [[TMP83]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP229:%.*]] = fadd fast <8 x float> [[TMP99]], [[VEC_PHI2]]
-; CHECK-NEXT: [[TMP230:%.*]] = fadd fast <8 x float> [[TMP115]], [[VEC_PHI3]]
-; CHECK-NEXT: [[TMP231:%.*]] = fadd fast <8 x float> [[TMP131]], [[VEC_PHI4]]
-; CHECK-NEXT: [[TMP232]] = fadd fast <8 x float> [[TMP228]], [[TMP179]]
-; CHECK-NEXT: [[TMP233]] = fadd fast <8 x float> [[TMP229]], [[TMP195]]
-; CHECK-NEXT: [[TMP234]] = fadd fast <8 x float> [[TMP230]], [[TMP211]]
-; CHECK-NEXT: [[TMP235]] = fadd fast <8 x float> [[TMP231]], [[TMP227]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP112:%.*]] = insertelement <4 x float> poison, float [[TMP184]], i32 0
+; CHECK-NEXT: [[TMP113:%.*]] = insertelement <4 x float> [[TMP112]], float [[TMP185]], i32 1
+; CHECK-NEXT: [[TMP114:%.*]] = insertelement <4 x float> [[TMP113]], float [[TMP186]], i32 2
+; CHECK-NEXT: [[TMP115:%.*]] = insertelement <4 x float> [[TMP114]], float [[TMP187]], i32 3
+; CHECK-NEXT: [[TMP116:%.*]] = fadd fast <4 x float> [[TMP55]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP117:%.*]] = fadd fast <4 x float> [[TMP63]], [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP118:%.*]] = fadd fast <4 x float> [[TMP59]], [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP119:%.*]] = fadd fast <4 x float> [[TMP67]], [[VEC_PHI4]]
+; CHECK-NEXT: [[TMP120]] = fadd fast <4 x float> [[TMP116]], [[TMP95]]
+; CHECK-NEXT: [[TMP121]] = fadd fast <4 x float> [[TMP117]], [[TMP99]]
+; CHECK-NEXT: [[TMP122]] = fadd fast <4 x float> [[TMP118]], [[TMP107]]
+; CHECK-NEXT: [[TMP123]] = fadd fast <4 x float> [[TMP119]], [[TMP115]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP236:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP236]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP233]], [[TMP232]]
-; CHECK-NEXT: [[BIN_RDX5:%.*]] = fadd fast <8 x float> [[TMP234]], [[BIN_RDX]]
-; CHECK-NEXT: [[BIN_RDX6:%.*]] = fadd fast <8 x float> [[TMP235]], [[BIN_RDX5]]
-; CHECK-NEXT: [[TMP237:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[BIN_RDX6]])
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP121]], [[TMP120]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = fadd fast <4 x float> [[TMP122]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX6:%.*]] = fadd fast <4 x float> [[TMP123]], [[BIN_RDX5]]
+; CHECK-NEXT: [[TMP125:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX6]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
@@ -339,7 +227,7 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP237]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP125]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF7:%.*]] = urem i64 [[TMP2]], 4
; CHECK-NEXT: [[N_VEC8:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF7]]
; CHECK-NEXT: [[TMP238:%.*]] = shl i64 [[N_VEC8]], 5
@@ -907,6 +795,7 @@ define i32 @g(i64 %n) {
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF25:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index 8caab456c23fa..adf85ce532faf 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -344,39 +344,29 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
; CHECK-LABEL: @test_for_tried_to_force_scalar(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
-; CHECK-NEXT: [[CONFLICT_RDX20:%.*]] = icmp ule i64 [[TMP0]], 8
+; CHECK-NEXT: [[CONFLICT_RDX20:%.*]] = icmp ule i64 [[TMP0]], 4
; CHECK-NEXT: br i1 [[CONFLICT_RDX20]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 8, i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 4, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP4]]
+; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x ptr> poison, ptr [[A:%.*]], i32 3
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x ptr> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 6
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 7
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr nusw [3 x float], ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP13]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x ptr> [[TMP17]], ptr [[TMP14]], i32 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x ptr> [[TMP18]], ptr [[TMP15]], i32 2
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x ptr> [[TMP19]], ptr [[TMP16]], i32 3
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr nusw [3 x float], ptr [[A]], i64 [[TMP8]]
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP21]], i32 0
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x ptr> [[TMP25]], ptr [[TMP22]], i32 1
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x ptr> [[TMP26]], ptr [[TMP23]], i32 2
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i32 3
+; CHECK-NEXT: [[TMP28]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i32 3
; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x ptr> [[TMP20]], <4 x ptr> [[TMP28]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP21]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -385,7 +375,7 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x ptr> [[TMP29]], i64 3
; CHECK-NEXT: [[TMP36:%.*]] = load float, ptr [[TMP37]], align 4
; CHECK-NEXT: store float [[TMP36]], ptr [[B:%.*]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index 31a993a8f52cd..3dd496ab36cda 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -81,114 +81,9 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
; CHECK-LABEL: define void @geps_feeding_interleave_groups_with_reuse(
; CHECK-SAME: ptr [[ARG:%.*]], i64 [[ARG1:%.*]], ptr [[ARG2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[ARG1]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 36
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ARG]], i64 16
-; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[ARG1]])
-; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[ARG1]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 16
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[ARG2]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[ARG1]], 5
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 32
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[TMP7]]
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ARG2]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[ARG]], [[SCEVGEP1]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 5
-; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP8]], 5
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[TMP9]], 16
-; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[TMP10]], 16
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = load float, ptr [[TMP11]], align 4, !alias.scope [[META3:![0-9]+]]
-; CHECK-NEXT: [[TMP20:%.*]] = load float, ptr [[TMP12]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x float> poison, float [[TMP19]], i32 0
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x float> [[TMP21]], float [[TMP20]], i32 1
-; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP15]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[TMP16]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <2 x float> poison, float [[TMP23]], i32 0
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x float> [[TMP25]], float [[TMP24]], i32 1
-; CHECK-NEXT: [[TMP27:%.*]] = fadd <2 x float> [[TMP22]], [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = fmul <2 x float> [[TMP27]], zeroinitializer
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 4
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 4
-; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP29]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP32:%.*]] = load float, ptr [[TMP30]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <2 x float> poison, float [[TMP31]], i32 0
-; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x float> [[TMP33]], float [[TMP32]], i32 1
-; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 4
-; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 4
-; CHECK-NEXT: [[TMP37:%.*]] = load float, ptr [[TMP35]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP38:%.*]] = load float, ptr [[TMP36]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP39:%.*]] = insertelement <2 x float> poison, float [[TMP37]], i32 0
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <2 x float> [[TMP39]], float [[TMP38]], i32 1
-; CHECK-NEXT: [[TMP41:%.*]] = fadd <2 x float> [[TMP34]], [[TMP40]]
-; CHECK-NEXT: [[TMP42:%.*]] = fmul <2 x float> [[TMP41]], zeroinitializer
-; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 8
-; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 8
-; CHECK-NEXT: [[TMP45:%.*]] = load float, ptr [[TMP43]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP46:%.*]] = load float, ptr [[TMP44]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP47:%.*]] = insertelement <2 x float> poison, float [[TMP45]], i32 0
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <2 x float> [[TMP47]], float [[TMP46]], i32 1
-; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 8
-; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 8
-; CHECK-NEXT: [[TMP51:%.*]] = load float, ptr [[TMP49]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP52:%.*]] = load float, ptr [[TMP50]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP53:%.*]] = insertelement <2 x float> poison, float [[TMP51]], i32 0
-; CHECK-NEXT: [[TMP54:%.*]] = insertelement <2 x float> [[TMP53]], float [[TMP52]], i32 1
-; CHECK-NEXT: [[TMP55:%.*]] = fadd <2 x float> [[TMP48]], [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = fmul <2 x float> [[TMP55]], zeroinitializer
-; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 12
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 12
-; CHECK-NEXT: [[TMP59:%.*]] = load float, ptr [[TMP57]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP60:%.*]] = load float, ptr [[TMP58]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP61:%.*]] = insertelement <2 x float> poison, float [[TMP59]], i32 0
-; CHECK-NEXT: [[TMP62:%.*]] = insertelement <2 x float> [[TMP61]], float [[TMP60]], i32 1
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 12
-; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 12
-; CHECK-NEXT: [[TMP65:%.*]] = load float, ptr [[TMP63]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP66:%.*]] = load float, ptr [[TMP64]], align 4, !alias.scope [[META3]]
-; CHECK-NEXT: [[TMP67:%.*]] = insertelement <2 x float> poison, float [[TMP65]], i32 0
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <2 x float> [[TMP67]], float [[TMP66]], i32 1
-; CHECK-NEXT: [[TMP69:%.*]] = fadd <2 x float> [[TMP62]], [[TMP68]]
-; CHECK-NEXT: [[TMP70:%.*]] = fmul <2 x float> [[TMP69]], zeroinitializer
-; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <2 x float> [[TMP28]], <2 x float> [[TMP42]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <2 x float> [[TMP56]], <2 x float> [[TMP70]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <4 x float> [[TMP71]], <4 x float> [[TMP72]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP73]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP18]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP74:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP74]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[SHL_IV_5:%.*]] = shl i64 [[IV]], 5
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[SHL_IV_5]]
; CHECK-NEXT: [[ADD_5:%.*]] = or disjoint i64 [[SHL_IV_5]], 16
@@ -226,7 +121,7 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
; CHECK-NEXT: store float [[MUL_4]], ptr [[GEP_11]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[ARG1]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -397,38 +292,38 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[TMP51:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP52:%.*]] = lshr exact i64 [[TMP51]], 1
; CHECK-NEXT: [[TMP53:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[TMP52]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP53]], align 4, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP53]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP54:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[WIDE_GEP42:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP54]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP42]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP42]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP55:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[WIDE_GEP43:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP55]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC41]], <4 x ptr> align 4 [[WIDE_GEP43]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC41]], <4 x ptr> align 4 [[WIDE_GEP43]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP56:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: [[WIDE_GEP44:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP56]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP44]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP44]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP57:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[WIDE_GEP45:%.*]] = getelementptr i32, ptr [[B]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP45]], <4 x i1> splat (i1 true), <4 x i32> poison), !alias.scope [[META15:![0-9]+]], !noalias [[META13]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP45]], <4 x i1> splat (i1 true), <4 x i32> poison), !alias.scope [[META8:![0-9]+]], !noalias [[META6]]
; CHECK-NEXT: [[WIDE_GEP46:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP57]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[WIDE_MASKED_GATHER]], <4 x ptr> align 4 [[WIDE_GEP46]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[WIDE_MASKED_GATHER]], <4 x ptr> align 4 [[WIDE_GEP46]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP58:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[WIDE_GEP47:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP58]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP47]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP47]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP59:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 6)
; CHECK-NEXT: [[WIDE_GEP48:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP59]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP48]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP48]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP60:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 7)
; CHECK-NEXT: [[WIDE_GEP49:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP60]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP49]], <4 x i1> splat (i1 true)), !alias.scope [[META13]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP49]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 32)
; CHECK-NEXT: [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP61]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP61]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -469,7 +364,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: store i32 0, ptr [[GEP_A_7]], align 4
; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -665,15 +560,8 @@ attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="cascadelake" }
; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"}
; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
-; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], !"LVerDomain"}
-; CHECK: [[META13]] = !{[[META14:![0-9]+]]}
-; CHECK: [[META14]] = distinct !{[[META14]], [[META12]]}
-; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
-; CHECK: [[META16]] = distinct !{[[META16]], [[META12]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]]}
+; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META5]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index 79a5bbb75cae4..f22c508427351 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -6,28 +6,25 @@
define void @test_store_initially_interleave(i32 %n, ptr noalias %src) #0 {
; I64-LABEL: define void @test_store_initially_interleave(
; I64-SAME: i32 [[N:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
-; I64-NEXT: [[ITER_CHECK:.*:]]
+; I64-NEXT: [[ENTRY:.*:]]
; I64-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
-; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 4
-; I64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 8
+; I64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; I64: [[VECTOR_SCEVCHECK]]:
; I64-NEXT: [[TMP1:%.*]] = icmp slt i32 [[N]], 0
-; I64-NEXT: br i1 [[TMP1]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; I64: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; I64-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i32 [[TMP0]], 16
-; I64-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; I64-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; I64: [[VECTOR_PH]]:
-; I64-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 16
+; I64-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 8
; I64-NEXT: [[TMP2:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
-; I64-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 16, i32 [[N_MOD_VF]]
+; I64-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 8, i32 [[N_MOD_VF]]
; I64-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP3]]
; I64-NEXT: br label %[[VECTOR_BODY:.*]]
; I64: [[VECTOR_BODY]]:
; I64-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I64-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I64-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; I64-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i32> [[STEP_ADD]], splat (i32 4)
-; I64-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i32> [[STEP_ADD_2]], splat (i32 4)
+; I64-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; I64-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
+; I64-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i32> [[STEP_ADD]], splat (i32 2)
+; I64-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i32> [[STEP_ADD_2]], splat (i32 2)
; I64-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 1
; I64-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 2
; I64-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 3
@@ -35,153 +32,68 @@ define void @test_store_initially_interleave(i32 %n, ptr noalias %src) #0 {
; I64-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 5
; I64-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], 6
; I64-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], 7
-; I64-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 8
-; I64-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 9
-; I64-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 10
-; I64-NEXT: [[TMP14:%.*]] = add i32 [[INDEX]], 11
-; I64-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], 12
-; I64-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], 13
-; I64-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 14
-; I64-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], 15
-; I64-NEXT: [[TMP19:%.*]] = uitofp <4 x i32> [[VEC_IND]] to <4 x double>
-; I64-NEXT: [[TMP20:%.*]] = extractelement <4 x double> [[TMP19]], i64 0
-; I64-NEXT: [[TMP21:%.*]] = extractelement <4 x double> [[TMP19]], i64 1
-; I64-NEXT: [[TMP22:%.*]] = extractelement <4 x double> [[TMP19]], i64 2
-; I64-NEXT: [[TMP23:%.*]] = extractelement <4 x double> [[TMP19]], i64 3
-; I64-NEXT: [[TMP24:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
-; I64-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[TMP24]], i64 0
-; I64-NEXT: [[TMP26:%.*]] = extractelement <4 x double> [[TMP24]], i64 1
-; I64-NEXT: [[TMP27:%.*]] = extractelement <4 x double> [[TMP24]], i64 2
-; I64-NEXT: [[TMP28:%.*]] = extractelement <4 x double> [[TMP24]], i64 3
-; I64-NEXT: [[TMP29:%.*]] = uitofp <4 x i32> [[STEP_ADD_2]] to <4 x double>
-; I64-NEXT: [[TMP30:%.*]] = extractelement <4 x double> [[TMP29]], i64 0
-; I64-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[TMP29]], i64 1
-; I64-NEXT: [[TMP32:%.*]] = extractelement <4 x double> [[TMP29]], i64 2
-; I64-NEXT: [[TMP33:%.*]] = extractelement <4 x double> [[TMP29]], i64 3
-; I64-NEXT: [[TMP34:%.*]] = uitofp <4 x i32> [[STEP_ADD_3]] to <4 x double>
-; I64-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[TMP34]], i64 0
-; I64-NEXT: [[TMP36:%.*]] = extractelement <4 x double> [[TMP34]], i64 1
-; I64-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP34]], i64 2
-; I64-NEXT: [[TMP38:%.*]] = extractelement <4 x double> [[TMP34]], i64 3
-; I64-NEXT: [[TMP39:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
-; I64-NEXT: [[TMP40:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
-; I64-NEXT: [[TMP41:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
-; I64-NEXT: [[TMP42:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
-; I64-NEXT: [[TMP43:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
-; I64-NEXT: [[TMP44:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
-; I64-NEXT: [[TMP45:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
-; I64-NEXT: [[TMP46:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP10]]
-; I64-NEXT: [[TMP47:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP11]]
-; I64-NEXT: [[TMP48:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP12]]
-; I64-NEXT: [[TMP49:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP13]]
-; I64-NEXT: [[TMP50:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP14]]
-; I64-NEXT: [[TMP51:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP15]]
-; I64-NEXT: [[TMP52:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP16]]
-; I64-NEXT: [[TMP53:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP17]]
-; I64-NEXT: [[TMP54:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP18]]
-; I64-NEXT: [[TMP55:%.*]] = load ptr, ptr [[TMP39]], align 4
-; I64-NEXT: [[TMP56:%.*]] = load ptr, ptr [[TMP40]], align 4
-; I64-NEXT: [[TMP57:%.*]] = load ptr, ptr [[TMP41]], align 4
-; I64-NEXT: [[TMP58:%.*]] = load ptr, ptr [[TMP42]], align 4
-; I64-NEXT: [[TMP59:%.*]] = load ptr, ptr [[TMP43]], align 4
-; I64-NEXT: [[TMP60:%.*]] = load ptr, ptr [[TMP44]], align 4
-; I64-NEXT: [[TMP61:%.*]] = load ptr, ptr [[TMP45]], align 4
-; I64-NEXT: [[TMP62:%.*]] = load ptr, ptr [[TMP46]], align 4
-; I64-NEXT: [[TMP63:%.*]] = load ptr, ptr [[TMP47]], align 4
-; I64-NEXT: [[TMP64:%.*]] = load ptr, ptr [[TMP48]], align 4
-; I64-NEXT: [[TMP65:%.*]] = load ptr, ptr [[TMP49]], align 4
-; I64-NEXT: [[TMP66:%.*]] = load ptr, ptr [[TMP50]], align 4
-; I64-NEXT: [[TMP67:%.*]] = load ptr, ptr [[TMP51]], align 4
-; I64-NEXT: [[TMP68:%.*]] = load ptr, ptr [[TMP52]], align 4
-; I64-NEXT: [[TMP69:%.*]] = load ptr, ptr [[TMP53]], align 4
-; I64-NEXT: [[TMP70:%.*]] = load ptr, ptr [[TMP54]], align 4
-; I64-NEXT: store double [[TMP20]], ptr [[TMP55]], align 4
-; I64-NEXT: store double [[TMP21]], ptr [[TMP56]], align 4
-; I64-NEXT: store double [[TMP22]], ptr [[TMP57]], align 4
-; I64-NEXT: store double [[TMP23]], ptr [[TMP58]], align 4
-; I64-NEXT: store double [[TMP25]], ptr [[TMP59]], align 4
-; I64-NEXT: store double [[TMP26]], ptr [[TMP60]], align 4
-; I64-NEXT: store double [[TMP27]], ptr [[TMP61]], align 4
-; I64-NEXT: store double [[TMP28]], ptr [[TMP62]], align 4
-; I64-NEXT: store double [[TMP30]], ptr [[TMP63]], align 4
-; I64-NEXT: store double [[TMP31]], ptr [[TMP64]], align 4
-; I64-NEXT: store double [[TMP32]], ptr [[TMP65]], align 4
-; I64-NEXT: store double [[TMP33]], ptr [[TMP66]], align 4
-; I64-NEXT: store double [[TMP35]], ptr [[TMP67]], align 4
-; I64-NEXT: store double [[TMP36]], ptr [[TMP68]], align 4
-; I64-NEXT: store double [[TMP37]], ptr [[TMP69]], align 4
-; I64-NEXT: store double [[TMP38]], ptr [[TMP70]], align 4
-; I64-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; I64-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
-; I64-NEXT: [[TMP71:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[TMP71]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; I64-NEXT: [[TMP11:%.*]] = uitofp <2 x i32> [[VEC_IND]] to <2 x double>
+; I64-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP11]], i64 0
+; I64-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[TMP11]], i64 1
+; I64-NEXT: [[TMP14:%.*]] = uitofp <2 x i32> [[STEP_ADD]] to <2 x double>
+; I64-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP14]], i64 0
+; I64-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP14]], i64 1
+; I64-NEXT: [[TMP17:%.*]] = uitofp <2 x i32> [[STEP_ADD_2]] to <2 x double>
+; I64-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
+; I64-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
+; I64-NEXT: [[TMP20:%.*]] = uitofp <2 x i32> [[STEP_ADD_3]] to <2 x double>
+; I64-NEXT: [[TMP21:%.*]] = extractelement <2 x double> [[TMP20]], i64 0
+; I64-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP20]], i64 1
+; I64-NEXT: [[TMP23:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
+; I64-NEXT: [[TMP24:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
+; I64-NEXT: [[TMP25:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
+; I64-NEXT: [[TMP26:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
+; I64-NEXT: [[TMP27:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
+; I64-NEXT: [[TMP28:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
+; I64-NEXT: [[TMP29:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
+; I64-NEXT: [[TMP30:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP10]]
+; I64-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP23]], align 4
+; I64-NEXT: [[TMP32:%.*]] = load ptr, ptr [[TMP24]], align 4
+; I64-NEXT: [[TMP33:%.*]] = load ptr, ptr [[TMP25]], align 4
+; I64-NEXT: [[TMP34:%.*]] = load ptr, ptr [[TMP26]], align 4
+; I64-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP27]], align 4
+; I64-NEXT: [[TMP36:%.*]] = load ptr, ptr [[TMP28]], align 4
+; I64-NEXT: [[TMP37:%.*]] = load ptr, ptr [[TMP29]], align 4
+; I64-NEXT: [[TMP38:%.*]] = load ptr, ptr [[TMP30]], align 4
+; I64-NEXT: store double [[TMP12]], ptr [[TMP31]], align 4
+; I64-NEXT: store double [[TMP13]], ptr [[TMP32]], align 4
+; I64-NEXT: store double [[TMP15]], ptr [[TMP33]], align 4
+; I64-NEXT: store double [[TMP16]], ptr [[TMP34]], align 4
+; I64-NEXT: store double [[TMP18]], ptr [[TMP35]], align 4
+; I64-NEXT: store double [[TMP19]], ptr [[TMP36]], align 4
+; I64-NEXT: store double [[TMP21]], ptr [[TMP37]], align 4
+; I64-NEXT: store double [[TMP22]], ptr [[TMP38]], align 4
+; I64-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; I64-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD_3]], splat (i32 2)
+; I64-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; I64-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
-; I64-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; I64: [[VEC_EPILOG_ITER_CHECK]]:
-; I64-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP3]], 4
-; I64-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; I64: [[VEC_EPILOG_PH]]:
-; I64-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; I64-NEXT: [[N_MOD_VF2:%.*]] = urem i32 [[TMP0]], 4
-; I64-NEXT: [[TMP72:%.*]] = icmp eq i32 [[N_MOD_VF2]], 0
-; I64-NEXT: [[TMP73:%.*]] = select i1 [[TMP72]], i32 4, i32 [[N_MOD_VF2]]
-; I64-NEXT: [[N_VEC3:%.*]] = sub i32 [[TMP0]], [[TMP73]]
-; I64-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; I64-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; I64-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
-; I64-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; I64: [[VEC_EPILOG_VECTOR_BODY]]:
-; I64-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I64-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I64-NEXT: [[TMP74:%.*]] = add i32 [[INDEX4]], 1
-; I64-NEXT: [[TMP75:%.*]] = add i32 [[INDEX4]], 2
-; I64-NEXT: [[TMP76:%.*]] = add i32 [[INDEX4]], 3
-; I64-NEXT: [[TMP77:%.*]] = uitofp <4 x i32> [[VEC_IND5]] to <4 x double>
-; I64-NEXT: [[TMP78:%.*]] = extractelement <4 x double> [[TMP77]], i64 0
-; I64-NEXT: [[TMP79:%.*]] = extractelement <4 x double> [[TMP77]], i64 1
-; I64-NEXT: [[TMP80:%.*]] = extractelement <4 x double> [[TMP77]], i64 2
-; I64-NEXT: [[TMP81:%.*]] = extractelement <4 x double> [[TMP77]], i64 3
-; I64-NEXT: [[TMP82:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX4]]
-; I64-NEXT: [[TMP83:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP74]]
-; I64-NEXT: [[TMP84:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP75]]
-; I64-NEXT: [[TMP85:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP76]]
-; I64-NEXT: [[TMP86:%.*]] = load ptr, ptr [[TMP82]], align 4
-; I64-NEXT: [[TMP87:%.*]] = load ptr, ptr [[TMP83]], align 4
-; I64-NEXT: [[TMP88:%.*]] = load ptr, ptr [[TMP84]], align 4
-; I64-NEXT: [[TMP89:%.*]] = load ptr, ptr [[TMP85]], align 4
-; I64-NEXT: store double [[TMP78]], ptr [[TMP86]], align 4
-; I64-NEXT: store double [[TMP79]], ptr [[TMP87]], align 4
-; I64-NEXT: store double [[TMP80]], ptr [[TMP88]], align 4
-; I64-NEXT: store double [[TMP81]], ptr [[TMP89]], align 4
-; I64-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX4]], 4
-; I64-NEXT: [[VEC_IND_NEXT7]] = add <4 x i32> [[VEC_IND5]], splat (i32 4)
-; I64-NEXT: [[TMP90:%.*]] = icmp eq i32 [[INDEX_NEXT6]], [[N_VEC3]]
-; I64-NEXT: br i1 [[TMP90]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; I64: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; I64-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
-; I64: [[VEC_EPILOG_SCALAR_PH]]:
+; I64-NEXT: br label %[[SCALAR_PH]]
+; I64: [[SCALAR_PH]]:
;
; I32-LABEL: define void @test_store_initially_interleave(
; I32-SAME: i32 [[N:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
-; I32-NEXT: [[ITER_CHECK:.*:]]
+; I32-NEXT: [[ENTRY:.*:]]
; I32-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
-; I32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 4
-; I32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; I32: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; I32-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i32 [[TMP0]], 16
-; I32-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; I32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 8
+; I32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; I32: [[VECTOR_PH]]:
-; I32-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 16
+; I32-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 8
; I32-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
-; I32-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 16, i32 [[N_MOD_VF]]
+; I32-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 8, i32 [[N_MOD_VF]]
; I32-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]
; I32-NEXT: br label %[[VECTOR_BODY:.*]]
; I32: [[VECTOR_BODY]]:
; I32-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I32-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I32-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; I32-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i32> [[STEP_ADD]], splat (i32 4)
-; I32-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i32> [[STEP_ADD_2]], splat (i32 4)
+; I32-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; I32-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i32> [[VEC_IND]], splat (i32 2)
+; I32-NEXT: [[STEP_ADD_2:%.*]] = add nuw <2 x i32> [[STEP_ADD]], splat (i32 2)
+; I32-NEXT: [[STEP_ADD_3:%.*]] = add nuw <2 x i32> [[STEP_ADD_2]], splat (i32 2)
; I32-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 1
; I32-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 2
; I32-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 3
@@ -189,131 +101,49 @@ define void @test_store_initially_interleave(i32 %n, ptr noalias %src) #0 {
; I32-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 5
; I32-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 6
; I32-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], 7
-; I32-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], 8
-; I32-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 9
-; I32-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 10
-; I32-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 11
-; I32-NEXT: [[TMP14:%.*]] = add i32 [[INDEX]], 12
-; I32-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], 13
-; I32-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], 14
-; I32-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 15
-; I32-NEXT: [[TMP18:%.*]] = uitofp <4 x i32> [[VEC_IND]] to <4 x double>
-; I32-NEXT: [[TMP19:%.*]] = extractelement <4 x double> [[TMP18]], i64 0
-; I32-NEXT: [[TMP20:%.*]] = extractelement <4 x double> [[TMP18]], i64 1
-; I32-NEXT: [[TMP21:%.*]] = extractelement <4 x double> [[TMP18]], i64 2
-; I32-NEXT: [[TMP22:%.*]] = extractelement <4 x double> [[TMP18]], i64 3
-; I32-NEXT: [[TMP23:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
-; I32-NEXT: [[TMP24:%.*]] = extractelement <4 x double> [[TMP23]], i64 0
-; I32-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[TMP23]], i64 1
-; I32-NEXT: [[TMP26:%.*]] = extractelement <4 x double> [[TMP23]], i64 2
-; I32-NEXT: [[TMP27:%.*]] = extractelement <4 x double> [[TMP23]], i64 3
-; I32-NEXT: [[TMP28:%.*]] = uitofp <4 x i32> [[STEP_ADD_2]] to <4 x double>
-; I32-NEXT: [[TMP29:%.*]] = extractelement <4 x double> [[TMP28]], i64 0
-; I32-NEXT: [[TMP30:%.*]] = extractelement <4 x double> [[TMP28]], i64 1
-; I32-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[TMP28]], i64 2
-; I32-NEXT: [[TMP32:%.*]] = extractelement <4 x double> [[TMP28]], i64 3
-; I32-NEXT: [[TMP33:%.*]] = uitofp <4 x i32> [[STEP_ADD_3]] to <4 x double>
-; I32-NEXT: [[TMP34:%.*]] = extractelement <4 x double> [[TMP33]], i64 0
-; I32-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[TMP33]], i64 1
-; I32-NEXT: [[TMP36:%.*]] = extractelement <4 x double> [[TMP33]], i64 2
-; I32-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP33]], i64 3
-; I32-NEXT: [[TMP38:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
-; I32-NEXT: [[TMP39:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP3]]
-; I32-NEXT: [[TMP40:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
-; I32-NEXT: [[TMP41:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
-; I32-NEXT: [[TMP42:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
-; I32-NEXT: [[TMP43:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
-; I32-NEXT: [[TMP44:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
-; I32-NEXT: [[TMP45:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
-; I32-NEXT: [[TMP46:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP10]]
-; I32-NEXT: [[TMP47:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP11]]
-; I32-NEXT: [[TMP48:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP12]]
-; I32-NEXT: [[TMP49:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP13]]
-; I32-NEXT: [[TMP50:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP14]]
-; I32-NEXT: [[TMP51:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP15]]
-; I32-NEXT: [[TMP52:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP16]]
-; I32-NEXT: [[TMP53:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP17]]
-; I32-NEXT: [[TMP54:%.*]] = load ptr, ptr [[TMP38]], align 4
-; I32-NEXT: [[TMP55:%.*]] = load ptr, ptr [[TMP39]], align 4
-; I32-NEXT: [[TMP56:%.*]] = load ptr, ptr [[TMP40]], align 4
-; I32-NEXT: [[TMP57:%.*]] = load ptr, ptr [[TMP41]], align 4
-; I32-NEXT: [[TMP58:%.*]] = load ptr, ptr [[TMP42]], align 4
-; I32-NEXT: [[TMP59:%.*]] = load ptr, ptr [[TMP43]], align 4
-; I32-NEXT: [[TMP60:%.*]] = load ptr, ptr [[TMP44]], align 4
-; I32-NEXT: [[TMP61:%.*]] = load ptr, ptr [[TMP45]], align 4
-; I32-NEXT: [[TMP62:%.*]] = load ptr, ptr [[TMP46]], align 4
-; I32-NEXT: [[TMP63:%.*]] = load ptr, ptr [[TMP47]], align 4
-; I32-NEXT: [[TMP64:%.*]] = load ptr, ptr [[TMP48]], align 4
-; I32-NEXT: [[TMP65:%.*]] = load ptr, ptr [[TMP49]], align 4
-; I32-NEXT: [[TMP66:%.*]] = load ptr, ptr [[TMP50]], align 4
-; I32-NEXT: [[TMP67:%.*]] = load ptr, ptr [[TMP51]], align 4
-; I32-NEXT: [[TMP68:%.*]] = load ptr, ptr [[TMP52]], align 4
-; I32-NEXT: [[TMP69:%.*]] = load ptr, ptr [[TMP53]], align 4
-; I32-NEXT: store double [[TMP19]], ptr [[TMP54]], align 4
-; I32-NEXT: store double [[TMP20]], ptr [[TMP55]], align 4
-; I32-NEXT: store double [[TMP21]], ptr [[TMP56]], align 4
-; I32-NEXT: store double [[TMP22]], ptr [[TMP57]], align 4
-; I32-NEXT: store double [[TMP24]], ptr [[TMP58]], align 4
-; I32-NEXT: store double [[TMP25]], ptr [[TMP59]], align 4
-; I32-NEXT: store double [[TMP26]], ptr [[TMP60]], align 4
-; I32-NEXT: store double [[TMP27]], ptr [[TMP61]], align 4
-; I32-NEXT: store double [[TMP29]], ptr [[TMP62]], align 4
-; I32-NEXT: store double [[TMP30]], ptr [[TMP63]], align 4
-; I32-NEXT: store double [[TMP31]], ptr [[TMP64]], align 4
-; I32-NEXT: store double [[TMP32]], ptr [[TMP65]], align 4
-; I32-NEXT: store double [[TMP34]], ptr [[TMP66]], align 4
-; I32-NEXT: store double [[TMP35]], ptr [[TMP67]], align 4
-; I32-NEXT: store double [[TMP36]], ptr [[TMP68]], align 4
-; I32-NEXT: store double [[TMP37]], ptr [[TMP69]], align 4
-; I32-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; I32-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
-; I32-NEXT: [[TMP70:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP70]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; I32-NEXT: [[TMP10:%.*]] = uitofp <2 x i32> [[VEC_IND]] to <2 x double>
+; I32-NEXT: [[TMP11:%.*]] = extractelement <2 x double> [[TMP10]], i64 0
+; I32-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP10]], i64 1
+; I32-NEXT: [[TMP13:%.*]] = uitofp <2 x i32> [[STEP_ADD]] to <2 x double>
+; I32-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP13]], i64 0
+; I32-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP13]], i64 1
+; I32-NEXT: [[TMP16:%.*]] = uitofp <2 x i32> [[STEP_ADD_2]] to <2 x double>
+; I32-NEXT: [[TMP17:%.*]] = extractelement <2 x double> [[TMP16]], i64 0
+; I32-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP16]], i64 1
+; I32-NEXT: [[TMP19:%.*]] = uitofp <2 x i32> [[STEP_ADD_3]] to <2 x double>
+; I32-NEXT: [[TMP20:%.*]] = extractelement <2 x double> [[TMP19]], i64 0
+; I32-NEXT: [[TMP21:%.*]] = extractelement <2 x double> [[TMP19]], i64 1
+; I32-NEXT: [[TMP22:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX]]
+; I32-NEXT: [[TMP23:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP3]]
+; I32-NEXT: [[TMP24:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP4]]
+; I32-NEXT: [[TMP25:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP5]]
+; I32-NEXT: [[TMP26:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP6]]
+; I32-NEXT: [[TMP27:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP7]]
+; I32-NEXT: [[TMP28:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP8]]
+; I32-NEXT: [[TMP29:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP9]]
+; I32-NEXT: [[TMP30:%.*]] = load ptr, ptr [[TMP22]], align 4
+; I32-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP23]], align 4
+; I32-NEXT: [[TMP32:%.*]] = load ptr, ptr [[TMP24]], align 4
+; I32-NEXT: [[TMP33:%.*]] = load ptr, ptr [[TMP25]], align 4
+; I32-NEXT: [[TMP34:%.*]] = load ptr, ptr [[TMP26]], align 4
+; I32-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP27]], align 4
+; I32-NEXT: [[TMP36:%.*]] = load ptr, ptr [[TMP28]], align 4
+; I32-NEXT: [[TMP37:%.*]] = load ptr, ptr [[TMP29]], align 4
+; I32-NEXT: store double [[TMP11]], ptr [[TMP30]], align 4
+; I32-NEXT: store double [[TMP12]], ptr [[TMP31]], align 4
+; I32-NEXT: store double [[TMP14]], ptr [[TMP32]], align 4
+; I32-NEXT: store double [[TMP15]], ptr [[TMP33]], align 4
+; I32-NEXT: store double [[TMP17]], ptr [[TMP34]], align 4
+; I32-NEXT: store double [[TMP18]], ptr [[TMP35]], align 4
+; I32-NEXT: store double [[TMP20]], ptr [[TMP36]], align 4
+; I32-NEXT: store double [[TMP21]], ptr [[TMP37]], align 4
+; I32-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; I32-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD_3]], splat (i32 2)
+; I32-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; I32-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
-; I32-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; I32: [[VEC_EPILOG_ITER_CHECK]]:
-; I32-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP2]], 4
-; I32-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; I32: [[VEC_EPILOG_PH]]:
-; I32-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; I32-NEXT: [[N_MOD_VF2:%.*]] = urem i32 [[TMP0]], 4
-; I32-NEXT: [[TMP71:%.*]] = icmp eq i32 [[N_MOD_VF2]], 0
-; I32-NEXT: [[TMP72:%.*]] = select i1 [[TMP71]], i32 4, i32 [[N_MOD_VF2]]
-; I32-NEXT: [[N_VEC3:%.*]] = sub i32 [[TMP0]], [[TMP72]]
-; I32-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; I32-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; I32-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
-; I32-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; I32: [[VEC_EPILOG_VECTOR_BODY]]:
-; I32-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I32-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; I32-NEXT: [[TMP73:%.*]] = add i32 [[INDEX4]], 1
-; I32-NEXT: [[TMP74:%.*]] = add i32 [[INDEX4]], 2
-; I32-NEXT: [[TMP75:%.*]] = add i32 [[INDEX4]], 3
-; I32-NEXT: [[TMP76:%.*]] = uitofp <4 x i32> [[VEC_IND5]] to <4 x double>
-; I32-NEXT: [[TMP77:%.*]] = extractelement <4 x double> [[TMP76]], i64 0
-; I32-NEXT: [[TMP78:%.*]] = extractelement <4 x double> [[TMP76]], i64 1
-; I32-NEXT: [[TMP79:%.*]] = extractelement <4 x double> [[TMP76]], i64 2
-; I32-NEXT: [[TMP80:%.*]] = extractelement <4 x double> [[TMP76]], i64 3
-; I32-NEXT: [[TMP81:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[INDEX4]]
-; I32-NEXT: [[TMP82:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP73]]
-; I32-NEXT: [[TMP83:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP74]]
-; I32-NEXT: [[TMP84:%.*]] = getelementptr nusw { ptr, ptr, ptr }, ptr null, i32 [[TMP75]]
-; I32-NEXT: [[TMP85:%.*]] = load ptr, ptr [[TMP81]], align 4
-; I32-NEXT: [[TMP86:%.*]] = load ptr, ptr [[TMP82]], align 4
-; I32-NEXT: [[TMP87:%.*]] = load ptr, ptr [[TMP83]], align 4
-; I32-NEXT: [[TMP88:%.*]] = load ptr, ptr [[TMP84]], align 4
-; I32-NEXT: store double [[TMP77]], ptr [[TMP85]], align 4
-; I32-NEXT: store double [[TMP78]], ptr [[TMP86]], align 4
-; I32-NEXT: store double [[TMP79]], ptr [[TMP87]], align 4
-; I32-NEXT: store double [[TMP80]], ptr [[TMP88]], align 4
-; I32-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX4]], 4
-; I32-NEXT: [[VEC_IND_NEXT7]] = add <4 x i32> [[VEC_IND5]], splat (i32 4)
-; I32-NEXT: [[TMP89:%.*]] = icmp eq i32 [[INDEX_NEXT6]], [[N_VEC3]]
-; I32-NEXT: br i1 [[TMP89]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; I32: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; I32-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
-; I32: [[VEC_EPILOG_SCALAR_PH]]:
+; I32-NEXT: br label %[[SCALAR_PH]]
+; I32: [[SCALAR_PH]]:
;
entry:
br label %loop
@@ -340,10 +170,10 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: br i1 [[PRE]], [[EXIT:label %.*]], label %[[PH:.*]]
; I64: [[PH]]:
; I64-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64
-; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 4
+; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 8
; I64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; I64: [[VECTOR_PH]]:
-; I64-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 4
+; I64-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 8
; I64-NEXT: [[N_VEC:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF]]
; I64-NEXT: br label %[[VECTOR_BODY:.*]]
; I64: [[VECTOR_BODY]]:
@@ -351,29 +181,53 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; I64-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
; I64-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; I64-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; I64-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
-; I64-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
-; I64-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; I64-NEXT: [[TMP7:%.*]] = load double, ptr [[TMP3]], align 8
-; I64-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP4]], align 8
-; I64-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP5]], align 8
-; I64-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP6]], align 8
-; I64-NEXT: [[TMP11:%.*]] = shl i64 [[INDEX]], 1
-; I64-NEXT: [[TMP12:%.*]] = shl i64 [[TMP0]], 1
-; I64-NEXT: [[TMP13:%.*]] = shl i64 [[TMP1]], 1
-; I64-NEXT: [[TMP14:%.*]] = shl i64 [[TMP2]], 1
-; I64-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
-; I64-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]
-; I64-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
-; I64-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]
-; I64-NEXT: store double [[TMP7]], ptr [[TMP15]], align 8
-; I64-NEXT: store double [[TMP8]], ptr [[TMP16]], align 8
-; I64-NEXT: store double [[TMP9]], ptr [[TMP17]], align 8
-; I64-NEXT: store double [[TMP10]], ptr [[TMP18]], align 8
-; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; I64-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; I64-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
+; I64-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
+; I64-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
+; I64-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
+; I64-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; I64-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
+; I64-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
+; I64-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; I64-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
+; I64-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
+; I64-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; I64-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
+; I64-NEXT: [[TMP15:%.*]] = load double, ptr [[TMP7]], align 8
+; I64-NEXT: [[TMP16:%.*]] = load double, ptr [[TMP8]], align 8
+; I64-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP9]], align 8
+; I64-NEXT: [[TMP18:%.*]] = load double, ptr [[TMP10]], align 8
+; I64-NEXT: [[TMP19:%.*]] = load double, ptr [[TMP11]], align 8
+; I64-NEXT: [[TMP20:%.*]] = load double, ptr [[TMP12]], align 8
+; I64-NEXT: [[TMP21:%.*]] = load double, ptr [[TMP13]], align 8
+; I64-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP14]], align 8
+; I64-NEXT: [[TMP23:%.*]] = shl i64 [[INDEX]], 1
+; I64-NEXT: [[TMP24:%.*]] = shl i64 [[TMP0]], 1
+; I64-NEXT: [[TMP25:%.*]] = shl i64 [[TMP1]], 1
+; I64-NEXT: [[TMP26:%.*]] = shl i64 [[TMP2]], 1
+; I64-NEXT: [[TMP27:%.*]] = shl i64 [[TMP3]], 1
+; I64-NEXT: [[TMP28:%.*]] = shl i64 [[TMP4]], 1
+; I64-NEXT: [[TMP29:%.*]] = shl i64 [[TMP5]], 1
+; I64-NEXT: [[TMP30:%.*]] = shl i64 [[TMP6]], 1
+; I64-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP23]]
+; I64-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP24]]
+; I64-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP25]]
+; I64-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP26]]
+; I64-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP27]]
+; I64-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP28]]
+; I64-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP29]]
+; I64-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP30]]
+; I64-NEXT: store double [[TMP15]], ptr [[TMP31]], align 8
+; I64-NEXT: store double [[TMP16]], ptr [[TMP32]], align 8
+; I64-NEXT: store double [[TMP17]], ptr [[TMP33]], align 8
+; I64-NEXT: store double [[TMP18]], ptr [[TMP34]], align 8
+; I64-NEXT: store double [[TMP19]], ptr [[TMP35]], align 8
+; I64-NEXT: store double [[TMP20]], ptr [[TMP36]], align 8
+; I64-NEXT: store double [[TMP21]], ptr [[TMP37]], align 8
+; I64-NEXT: store double [[TMP22]], ptr [[TMP38]], align 8
+; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; I64-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; I64-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
; I64-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
@@ -419,7 +273,7 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I32-NEXT: store double [[TMP10]], ptr [[TMP18]], align 8
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I32-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; I32-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
; I32-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
@@ -475,7 +329,7 @@ define double @test_load_used_by_other_load_scev(ptr %ptr.a, ptr %ptr.b, ptr %pt
; I64-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; I64-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; I64-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; I64-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; I64-NEXT: br label %[[SCALAR_PH:.*]]
@@ -801,7 +655,7 @@ define void @loaded_address_used_by_load_through_blend(i64 %start, ptr noalias %
; I32-NEXT: store float [[TMP82]], ptr [[TMP90]], align 4
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I32-NEXT: [[TMP91:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I32-NEXT: br i1 [[TMP91]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; I32-NEXT: br i1 [[TMP91]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; I32-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -841,119 +695,32 @@ exit:
define void @address_use_in_different_block(ptr noalias %dst, ptr %src.0, ptr %src.1, i32 %x) #0 {
; I64-LABEL: define void @address_use_in_different_block(
; I64-SAME: ptr noalias [[DST:%.*]], ptr [[SRC_0:%.*]], ptr [[SRC_1:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
-; I64-NEXT: [[ENTRY:.*:]]
+; I64-NEXT: [[ENTRY:.*]]:
; I64-NEXT: [[X_POS:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 0)
; I64-NEXT: [[OFFSET:%.*]] = zext i32 [[X_POS]] to i64
-; I64-NEXT: br label %[[VECTOR_PH:.*]]
-; I64: [[VECTOR_PH]]:
-; I64-NEXT: br label %[[VECTOR_BODY:.*]]
-; I64: [[VECTOR_BODY]]:
-; I64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; I64-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; I64-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
-; I64-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; I64-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
-; I64-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
-; I64-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
-; I64-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
-; I64-NEXT: [[TMP7:%.*]] = mul i64 [[INDEX]], [[OFFSET]]
-; I64-NEXT: [[TMP8:%.*]] = mul i64 [[TMP0]], [[OFFSET]]
-; I64-NEXT: [[TMP9:%.*]] = mul i64 [[TMP1]], [[OFFSET]]
-; I64-NEXT: [[TMP10:%.*]] = mul i64 [[TMP2]], [[OFFSET]]
-; I64-NEXT: [[TMP11:%.*]] = mul i64 [[TMP3]], [[OFFSET]]
-; I64-NEXT: [[TMP12:%.*]] = mul i64 [[TMP4]], [[OFFSET]]
-; I64-NEXT: [[TMP13:%.*]] = mul i64 [[TMP5]], [[OFFSET]]
-; I64-NEXT: [[TMP14:%.*]] = mul i64 [[TMP6]], [[OFFSET]]
-; I64-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP7]]
-; I64-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP8]]
-; I64-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP9]]
-; I64-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP10]]
-; I64-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP11]]
-; I64-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP12]]
-; I64-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP13]]
-; I64-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP14]]
-; I64-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP15]], align 4
-; I64-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP16]], align 4
-; I64-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP17]], align 4
-; I64-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP18]], align 4
-; I64-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP19]], align 4
-; I64-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP20]], align 4
-; I64-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP21]], align 4
-; I64-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP22]], align 4
-; I64-NEXT: [[TMP31:%.*]] = sext i32 [[TMP23]] to i64
-; I64-NEXT: [[TMP32:%.*]] = sext i32 [[TMP24]] to i64
-; I64-NEXT: [[TMP33:%.*]] = sext i32 [[TMP25]] to i64
-; I64-NEXT: [[TMP34:%.*]] = sext i32 [[TMP26]] to i64
-; I64-NEXT: [[TMP35:%.*]] = sext i32 [[TMP27]] to i64
-; I64-NEXT: [[TMP36:%.*]] = sext i32 [[TMP28]] to i64
-; I64-NEXT: [[TMP37:%.*]] = sext i32 [[TMP29]] to i64
-; I64-NEXT: [[TMP38:%.*]] = sext i32 [[TMP30]] to i64
-; I64-NEXT: [[TMP39:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP31]]
-; I64-NEXT: [[TMP40:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP32]]
-; I64-NEXT: [[TMP41:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP33]]
-; I64-NEXT: [[TMP42:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP34]]
-; I64-NEXT: [[TMP43:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP35]]
-; I64-NEXT: [[TMP44:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP36]]
-; I64-NEXT: [[TMP45:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP37]]
-; I64-NEXT: [[TMP46:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[TMP38]]
-; I64-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP39]], i64 -8
-; I64-NEXT: [[TMP48:%.*]] = getelementptr i8, ptr [[TMP40]], i64 -8
-; I64-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[TMP41]], i64 -8
-; I64-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[TMP42]], i64 -8
-; I64-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[TMP43]], i64 -8
-; I64-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[TMP44]], i64 -8
-; I64-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[TMP45]], i64 -8
-; I64-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[TMP46]], i64 -8
-; I64-NEXT: [[TMP55:%.*]] = load double, ptr [[TMP47]], align 8
-; I64-NEXT: [[TMP56:%.*]] = load double, ptr [[TMP48]], align 8
-; I64-NEXT: [[TMP57:%.*]] = insertelement <2 x double> poison, double [[TMP55]], i32 0
-; I64-NEXT: [[TMP58:%.*]] = insertelement <2 x double> [[TMP57]], double [[TMP56]], i32 1
-; I64-NEXT: [[TMP59:%.*]] = load double, ptr [[TMP49]], align 8
-; I64-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP50]], align 8
-; I64-NEXT: [[TMP61:%.*]] = insertelement <2 x double> poison, double [[TMP59]], i32 0
-; I64-NEXT: [[TMP62:%.*]] = insertelement <2 x double> [[TMP61]], double [[TMP60]], i32 1
-; I64-NEXT: [[TMP63:%.*]] = load double, ptr [[TMP51]], align 8
-; I64-NEXT: [[TMP64:%.*]] = load double, ptr [[TMP52]], align 8
-; I64-NEXT: [[TMP65:%.*]] = insertelement <2 x double> poison, double [[TMP63]], i32 0
-; I64-NEXT: [[TMP66:%.*]] = insertelement <2 x double> [[TMP65]], double [[TMP64]], i32 1
-; I64-NEXT: [[TMP67:%.*]] = load double, ptr [[TMP53]], align 8
-; I64-NEXT: [[TMP68:%.*]] = load double, ptr [[TMP54]], align 8
-; I64-NEXT: [[TMP69:%.*]] = insertelement <2 x double> poison, double [[TMP67]], i32 0
-; I64-NEXT: [[TMP70:%.*]] = insertelement <2 x double> [[TMP69]], double [[TMP68]], i32 1
-; I64-NEXT: [[TMP71:%.*]] = fsub <2 x double> zeroinitializer, [[TMP58]]
-; I64-NEXT: [[TMP72:%.*]] = extractelement <2 x double> [[TMP71]], i64 0
-; I64-NEXT: [[TMP73:%.*]] = extractelement <2 x double> [[TMP71]], i64 1
-; I64-NEXT: [[TMP74:%.*]] = fsub <2 x double> zeroinitializer, [[TMP62]]
-; I64-NEXT: [[TMP75:%.*]] = extractelement <2 x double> [[TMP74]], i64 0
-; I64-NEXT: [[TMP76:%.*]] = extractelement <2 x double> [[TMP74]], i64 1
-; I64-NEXT: [[TMP77:%.*]] = fsub <2 x double> zeroinitializer, [[TMP66]]
-; I64-NEXT: [[TMP78:%.*]] = extractelement <2 x double> [[TMP77]], i64 0
-; I64-NEXT: [[TMP79:%.*]] = extractelement <2 x double> [[TMP77]], i64 1
-; I64-NEXT: [[TMP80:%.*]] = fsub <2 x double> zeroinitializer, [[TMP70]]
-; I64-NEXT: [[TMP81:%.*]] = extractelement <2 x double> [[TMP80]], i64 0
-; I64-NEXT: [[TMP82:%.*]] = extractelement <2 x double> [[TMP80]], i64 1
-; I64-NEXT: [[TMP83:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP7]]
-; I64-NEXT: [[TMP84:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP8]]
-; I64-NEXT: [[TMP85:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP9]]
-; I64-NEXT: [[TMP86:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP10]]
-; I64-NEXT: [[TMP87:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP11]]
-; I64-NEXT: [[TMP88:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP12]]
-; I64-NEXT: [[TMP89:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP13]]
-; I64-NEXT: [[TMP90:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP14]]
-; I64-NEXT: store double [[TMP72]], ptr [[TMP83]], align 8
-; I64-NEXT: store double [[TMP73]], ptr [[TMP84]], align 8
-; I64-NEXT: store double [[TMP75]], ptr [[TMP85]], align 8
-; I64-NEXT: store double [[TMP76]], ptr [[TMP86]], align 8
-; I64-NEXT: store double [[TMP78]], ptr [[TMP87]], align 8
-; I64-NEXT: store double [[TMP79]], ptr [[TMP88]], align 8
-; I64-NEXT: store double [[TMP81]], ptr [[TMP89]], align 8
-; I64-NEXT: store double [[TMP82]], ptr [[TMP90]], align 8
-; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; I64-NEXT: [[TMP91:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; I64-NEXT: br i1 [[TMP91]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; I64: [[MIDDLE_BLOCK]]:
-; I64-NEXT: br label %[[SCALAR_PH:.*]]
-; I64: [[SCALAR_PH]]:
+; I64-NEXT: br label %[[LOOP_HEADER:.*]]
+; I64: [[LOOP_HEADER]]:
+; I64-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; I64-NEXT: [[TMP0:%.*]] = mul i64 [[IV]], [[OFFSET]]
+; I64-NEXT: [[GEP_SRC_0:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[TMP0]]
+; I64-NEXT: [[L8:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4
+; I64-NEXT: [[C:%.*]] = icmp sgt i32 [[X]], 0
+; I64-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
+; I64: [[THEN]]:
+; I64-NEXT: br label %[[LOOP_LATCH]]
+; I64: [[LOOP_LATCH]]:
+; I64-NEXT: [[L_EXT:%.*]] = sext i32 [[L8]] to i64
+; I64-NEXT: [[GEP_SRC_1:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[L_EXT]]
+; I64-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[GEP_SRC_1]], i64 -8
+; I64-NEXT: [[L_2:%.*]] = load double, ptr [[TMP1]], align 8
+; I64-NEXT: [[SUB:%.*]] = fsub double 0.000000e+00, [[L_2]]
+; I64-NEXT: [[GEP_DST:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP0]]
+; I64-NEXT: store double [[SUB]], ptr [[GEP_DST]], align 8
+; I64-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; I64-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100
+; I64-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; I64: [[EXIT]]:
+; I64-NEXT: ret void
;
; I32-LABEL: define void @address_use_in_different_block(
; I32-SAME: ptr noalias [[DST:%.*]], ptr [[SRC_0:%.*]], ptr [[SRC_1:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
@@ -1015,7 +782,7 @@ define void @address_use_in_different_block(ptr noalias %dst, ptr %src.0, ptr %s
; I32-NEXT: store double [[TMP39]], ptr [[TMP43]], align 8
; I32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I32-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; I32-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; I32-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; I32: [[MIDDLE_BLOCK]]:
; I32-NEXT: br label %[[SCALAR_PH:.*]]
; I32: [[SCALAR_PH]]:
@@ -1139,7 +906,7 @@ define void @replicated_load_wide_store_derived_iv_zext_and(ptr noalias %src, pt
; I64-NEXT: store <4 x float> [[TMP62]], ptr [[TMP64]], align 4
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I64-NEXT: [[TMP65:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; I64-NEXT: br i1 [[TMP65]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; I64-NEXT: br i1 [[TMP65]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: br label %[[SCALAR_PH]]
; I64: [[SCALAR_PH]]:
@@ -1261,7 +1028,7 @@ define void @replicated_load_wide_store_derived_iv_zext_and2(ptr noalias %dst, p
; I64-NEXT: store <4 x float> [[TMP54]], ptr [[TMP56]], align 4
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I64-NEXT: [[TMP57:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; I64-NEXT: br i1 [[TMP57]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; I64-NEXT: br i1 [[TMP57]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: br label %[[SCALAR_PH]]
; I64: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 04ca0ec9d4c57..810e9cfd05bd4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -571,60 +571,28 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; MAX-BW-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
; MAX-BW-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
; MAX-BW-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
-; MAX-BW-NEXT: [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 32
-; MAX-BW-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 34
-; MAX-BW-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 36
-; MAX-BW-NEXT: [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], 38
-; MAX-BW-NEXT: [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 40
-; MAX-BW-NEXT: [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 42
-; MAX-BW-NEXT: [[TMP22:%.*]] = add i64 [[OFFSET_IDX]], 44
-; MAX-BW-NEXT: [[TMP23:%.*]] = add i64 [[OFFSET_IDX]], 46
-; MAX-BW-NEXT: [[TMP24:%.*]] = add i64 [[OFFSET_IDX]], 48
-; MAX-BW-NEXT: [[TMP25:%.*]] = add i64 [[OFFSET_IDX]], 50
-; MAX-BW-NEXT: [[TMP26:%.*]] = add i64 [[OFFSET_IDX]], 52
-; MAX-BW-NEXT: [[TMP27:%.*]] = add i64 [[OFFSET_IDX]], 54
-; MAX-BW-NEXT: [[TMP28:%.*]] = add i64 [[OFFSET_IDX]], 56
-; MAX-BW-NEXT: [[TMP29:%.*]] = add i64 [[OFFSET_IDX]], 58
-; MAX-BW-NEXT: [[TMP30:%.*]] = add i64 [[OFFSET_IDX]], 60
-; MAX-BW-NEXT: [[TMP31:%.*]] = add i64 [[OFFSET_IDX]], 62
; MAX-BW-NEXT: [[TMP32:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[OFFSET_IDX]]
-; MAX-BW-NEXT: [[WIDE_VEC:%.*]] = load <64 x i32>, ptr [[TMP32]], align 4
-; MAX-BW-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
-; MAX-BW-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
-; MAX-BW-NEXT: [[TMP33:%.*]] = add <32 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; MAX-BW-NEXT: [[TMP34:%.*]] = trunc <32 x i32> [[TMP33]] to <32 x i8>
-; MAX-BW-NEXT: [[TMP52:%.*]] = extractelement <32 x i8> [[TMP34]], i64 0
-; MAX-BW-NEXT: [[TMP53:%.*]] = extractelement <32 x i8> [[TMP34]], i64 1
-; MAX-BW-NEXT: [[TMP54:%.*]] = extractelement <32 x i8> [[TMP34]], i64 2
-; MAX-BW-NEXT: [[TMP55:%.*]] = extractelement <32 x i8> [[TMP34]], i64 3
-; MAX-BW-NEXT: [[TMP56:%.*]] = extractelement <32 x i8> [[TMP34]], i64 4
-; MAX-BW-NEXT: [[TMP57:%.*]] = extractelement <32 x i8> [[TMP34]], i64 5
-; MAX-BW-NEXT: [[TMP58:%.*]] = extractelement <32 x i8> [[TMP34]], i64 6
-; MAX-BW-NEXT: [[TMP59:%.*]] = extractelement <32 x i8> [[TMP34]], i64 7
-; MAX-BW-NEXT: [[TMP60:%.*]] = extractelement <32 x i8> [[TMP34]], i64 8
-; MAX-BW-NEXT: [[TMP61:%.*]] = extractelement <32 x i8> [[TMP34]], i64 9
-; MAX-BW-NEXT: [[TMP62:%.*]] = extractelement <32 x i8> [[TMP34]], i64 10
-; MAX-BW-NEXT: [[TMP63:%.*]] = extractelement <32 x i8> [[TMP34]], i64 11
-; MAX-BW-NEXT: [[TMP64:%.*]] = extractelement <32 x i8> [[TMP34]], i64 12
-; MAX-BW-NEXT: [[TMP65:%.*]] = extractelement <32 x i8> [[TMP34]], i64 13
-; MAX-BW-NEXT: [[TMP66:%.*]] = extractelement <32 x i8> [[TMP34]], i64 14
-; MAX-BW-NEXT: [[TMP67:%.*]] = extractelement <32 x i8> [[TMP34]], i64 15
-; MAX-BW-NEXT: [[TMP99:%.*]] = extractelement <32 x i8> [[TMP34]], i64 16
-; MAX-BW-NEXT: [[TMP100:%.*]] = extractelement <32 x i8> [[TMP34]], i64 17
-; MAX-BW-NEXT: [[TMP101:%.*]] = extractelement <32 x i8> [[TMP34]], i64 18
-; MAX-BW-NEXT: [[TMP102:%.*]] = extractelement <32 x i8> [[TMP34]], i64 19
-; MAX-BW-NEXT: [[TMP103:%.*]] = extractelement <32 x i8> [[TMP34]], i64 20
-; MAX-BW-NEXT: [[TMP104:%.*]] = extractelement <32 x i8> [[TMP34]], i64 21
-; MAX-BW-NEXT: [[TMP105:%.*]] = extractelement <32 x i8> [[TMP34]], i64 22
-; MAX-BW-NEXT: [[TMP106:%.*]] = extractelement <32 x i8> [[TMP34]], i64 23
-; MAX-BW-NEXT: [[TMP107:%.*]] = extractelement <32 x i8> [[TMP34]], i64 24
-; MAX-BW-NEXT: [[TMP108:%.*]] = extractelement <32 x i8> [[TMP34]], i64 25
-; MAX-BW-NEXT: [[TMP109:%.*]] = extractelement <32 x i8> [[TMP34]], i64 26
-; MAX-BW-NEXT: [[TMP110:%.*]] = extractelement <32 x i8> [[TMP34]], i64 27
-; MAX-BW-NEXT: [[TMP111:%.*]] = extractelement <32 x i8> [[TMP34]], i64 28
-; MAX-BW-NEXT: [[TMP112:%.*]] = extractelement <32 x i8> [[TMP34]], i64 29
-; MAX-BW-NEXT: [[TMP113:%.*]] = extractelement <32 x i8> [[TMP34]], i64 30
-; MAX-BW-NEXT: [[TMP114:%.*]] = extractelement <32 x i8> [[TMP34]], i64 31
+; MAX-BW-NEXT: [[WIDE_VEC:%.*]] = load <32 x i32>, ptr [[TMP32]], align 4
+; MAX-BW-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; MAX-BW-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; MAX-BW-NEXT: [[TMP17:%.*]] = add <16 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; MAX-BW-NEXT: [[TMP18:%.*]] = trunc <16 x i32> [[TMP17]] to <16 x i8>
+; MAX-BW-NEXT: [[TMP52:%.*]] = extractelement <16 x i8> [[TMP18]], i64 0
+; MAX-BW-NEXT: [[TMP53:%.*]] = extractelement <16 x i8> [[TMP18]], i64 1
+; MAX-BW-NEXT: [[TMP54:%.*]] = extractelement <16 x i8> [[TMP18]], i64 2
+; MAX-BW-NEXT: [[TMP55:%.*]] = extractelement <16 x i8> [[TMP18]], i64 3
+; MAX-BW-NEXT: [[TMP56:%.*]] = extractelement <16 x i8> [[TMP18]], i64 4
+; MAX-BW-NEXT: [[TMP57:%.*]] = extractelement <16 x i8> [[TMP18]], i64 5
+; MAX-BW-NEXT: [[TMP58:%.*]] = extractelement <16 x i8> [[TMP18]], i64 6
+; MAX-BW-NEXT: [[TMP59:%.*]] = extractelement <16 x i8> [[TMP18]], i64 7
+; MAX-BW-NEXT: [[TMP60:%.*]] = extractelement <16 x i8> [[TMP18]], i64 8
+; MAX-BW-NEXT: [[TMP61:%.*]] = extractelement <16 x i8> [[TMP18]], i64 9
+; MAX-BW-NEXT: [[TMP62:%.*]] = extractelement <16 x i8> [[TMP18]], i64 10
+; MAX-BW-NEXT: [[TMP63:%.*]] = extractelement <16 x i8> [[TMP18]], i64 11
+; MAX-BW-NEXT: [[TMP64:%.*]] = extractelement <16 x i8> [[TMP18]], i64 12
+; MAX-BW-NEXT: [[TMP65:%.*]] = extractelement <16 x i8> [[TMP18]], i64 13
+; MAX-BW-NEXT: [[TMP66:%.*]] = extractelement <16 x i8> [[TMP18]], i64 14
+; MAX-BW-NEXT: [[TMP67:%.*]] = extractelement <16 x i8> [[TMP18]], i64 15
; MAX-BW-NEXT: [[TMP69:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[OFFSET_IDX]]
; MAX-BW-NEXT: [[TMP70:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP1]]
; MAX-BW-NEXT: [[TMP71:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP2]]
@@ -641,22 +609,6 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; MAX-BW-NEXT: [[TMP82:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP13]]
; MAX-BW-NEXT: [[TMP83:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP14]]
; MAX-BW-NEXT: [[TMP51:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP15]]
-; MAX-BW-NEXT: [[TMP115:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP16]]
-; MAX-BW-NEXT: [[TMP84:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP17]]
-; MAX-BW-NEXT: [[TMP85:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP18]]
-; MAX-BW-NEXT: [[TMP86:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP19]]
-; MAX-BW-NEXT: [[TMP87:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP20]]
-; MAX-BW-NEXT: [[TMP88:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP21]]
-; MAX-BW-NEXT: [[TMP89:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP22]]
-; MAX-BW-NEXT: [[TMP90:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP23]]
-; MAX-BW-NEXT: [[TMP91:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP24]]
-; MAX-BW-NEXT: [[TMP92:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP25]]
-; MAX-BW-NEXT: [[TMP93:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP26]]
-; MAX-BW-NEXT: [[TMP94:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP27]]
-; MAX-BW-NEXT: [[TMP95:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP28]]
-; MAX-BW-NEXT: [[TMP96:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP29]]
-; MAX-BW-NEXT: [[TMP97:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP30]]
-; MAX-BW-NEXT: [[TMP98:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP31]]
; MAX-BW-NEXT: store i8 [[TMP52]], ptr [[TMP69]], align 1
; MAX-BW-NEXT: store i8 [[TMP53]], ptr [[TMP70]], align 1
; MAX-BW-NEXT: store i8 [[TMP54]], ptr [[TMP71]], align 1
@@ -673,23 +625,7 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; MAX-BW-NEXT: store i8 [[TMP65]], ptr [[TMP82]], align 1
; MAX-BW-NEXT: store i8 [[TMP66]], ptr [[TMP83]], align 1
; MAX-BW-NEXT: store i8 [[TMP67]], ptr [[TMP51]], align 1
-; MAX-BW-NEXT: store i8 [[TMP99]], ptr [[TMP115]], align 1
-; MAX-BW-NEXT: store i8 [[TMP100]], ptr [[TMP84]], align 1
-; MAX-BW-NEXT: store i8 [[TMP101]], ptr [[TMP85]], align 1
-; MAX-BW-NEXT: store i8 [[TMP102]], ptr [[TMP86]], align 1
-; MAX-BW-NEXT: store i8 [[TMP103]], ptr [[TMP87]], align 1
-; MAX-BW-NEXT: store i8 [[TMP104]], ptr [[TMP88]], align 1
-; MAX-BW-NEXT: store i8 [[TMP105]], ptr [[TMP89]], align 1
-; MAX-BW-NEXT: store i8 [[TMP106]], ptr [[TMP90]], align 1
-; MAX-BW-NEXT: store i8 [[TMP107]], ptr [[TMP91]], align 1
-; MAX-BW-NEXT: store i8 [[TMP108]], ptr [[TMP92]], align 1
-; MAX-BW-NEXT: store i8 [[TMP109]], ptr [[TMP93]], align 1
-; MAX-BW-NEXT: store i8 [[TMP110]], ptr [[TMP94]], align 1
-; MAX-BW-NEXT: store i8 [[TMP111]], ptr [[TMP95]], align 1
-; MAX-BW-NEXT: store i8 [[TMP112]], ptr [[TMP96]], align 1
-; MAX-BW-NEXT: store i8 [[TMP113]], ptr [[TMP97]], align 1
-; MAX-BW-NEXT: store i8 [[TMP114]], ptr [[TMP98]], align 1
-; MAX-BW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; MAX-BW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; MAX-BW-NEXT: [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
; MAX-BW-NEXT: br i1 [[TMP68]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; MAX-BW: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
index 9dc6b9bca174c..91907c3e4d69e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
@@ -121,7 +121,7 @@ define i8 @test_consecutive_ptr_load() readonly {
;; However, we should not take unconsecutive loads of pointers into account.
; CHECK: test_nonconsecutive_ptr_load
; CHECK: LV: The Smallest and Widest types: 16 / 64 bits.
-; CHECK: LV: Selecting VF: 4
+; CHECK: LV: Selecting VF: 1
define void @test_nonconsecutive_ptr_load() {
br label %1
>From f329c73b4192ee70f6a7d78cb4f7a8fc89980995 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 12 Jun 2026 11:57:06 +0000
Subject: [PATCH 3/4] Fix PhaseOrdering test
---
.../PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
index cc13db2b6f743..553fcc3b53f38 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes='default<O3>' -S %s | FileCheck %s
+; RUN: opt -passes='default<O3>' -force-vector-width=2 -S %s | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -12,7 +12,7 @@ define void @hoist_invariant_load(ptr %invariant_ptr, i64 %num_elements, ptr %ar
; CHECK-NEXT: [[CMP1_NOT:%.*]] = icmp eq i64 [[NUM_ELEMENTS]], 0
; CHECK-NEXT: br i1 [[CMP1_NOT]], label %[[EXIT:.*]], label %[[LOOP_LATCH_PREHEADER:.*]]
; CHECK: [[LOOP_LATCH_PREHEADER]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_ELEMENTS]], 11
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_ELEMENTS]], 5
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[LOOP_LATCH_PREHEADER6:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[NUM_ELEMENTS]], 5
>From 86b5f3f2e06b4030d3f04aefb50ecf29c0a59e74 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 12 Jun 2026 12:00:48 +0000
Subject: [PATCH 4/4] Address review comment
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3da35a49188a6..33777d9250314 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2981,7 +2981,7 @@ InstructionCost VPScalarIVStepsRecipe::computeCost(ElementCount VF,
// Determine the number of scalar adds we need to generate.
const unsigned NumLanes =
- vputils::onlyFirstLaneUsed(this) ? 1 : VF.getKnownMinValue();
+ vputils::onlyFirstLaneUsed(this) ? 1 : VF.getFixedValue();
// If there is no start index the first lane will be free, since
// add i32 x, 0 -> i32 x
More information about the llvm-commits
mailing list