[llvm] ab478d9 - [LV] Implement integer part of VPDerivedIV cost model (#198252)

[via llvm-commits]

Author: David Sherwood
Date: 2026-06-04T11:58:32+01:00
New Revision: ab478d9c57547e96e9b5db2621ac2d913bec6ab8

URL: https://github.com/llvm/llvm-project/commit/ab478d9c57547e96e9b5db2621ac2d913bec6ab8
DIFF: https://github.com/llvm/llvm-project/commit/ab478d9c57547e96e9b5db2621ac2d913bec6ab8.diff

LOG: [LV] Implement integer part of VPDerivedIV cost model (#198252)

Adds support for InductionDescriptor::IK_IntInduction variants of
inductions in VPDerivedIVRecipe::computeCost. Designing a cost model for
this recipe is tricky because it gets expanded in expandVPDerivedIV into
smaller individual recipes that will then be folded by simplifyRecipe.
Effectively the cost model has to guess how these will be folded
otherwise the costs will be too pessimistic. For example, 'add i32 %x,
0' and 'mul i32 %x, 1' should both be treated as zero cost since all
uses will be replaced with '%x'. Similarly,
'mul i32 %x, 4' will be converted to 'shl i32 %x, 2', and 'mul i32 %x,
-1' will be simplified to 'sub i32 0, %x'.

As can be seen by this PR, it's important to have non-zero costs for
cases where VPDerivedIVRecipe recipes will end up with non-foldable
instructions. Some of the tests now favour wider VFs to cover the
non-zero cost, or in other cases we prefer not to vectorise at all. I've
added loop attributes in some places to maintain the previous
vectorisation behaviour so that we're still matching the intent of the
test.

I built the LLVM test suite and I couldn't find any cases where we use
more than the first lane of the result so for now I maintained the
original behaviour and returned 0. The other induction types (pointers,
floating point values) are also left as returning 0 for now. These will
be handled in future PRs.

Added: 
    

Modified: 
    llvm/lib/Transforms/Vectorize/VPlan.h
    llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
    llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
    llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
    llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
    llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
    llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
    llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
    llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
    llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
    llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
    llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
    llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
    llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
    llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
    llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
    llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
    llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
    llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
    llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
    llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b740665fe70bf..46788a23eef99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4147,10 +4147,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
 
   /// Return the cost of this VPDerivedIVRecipe.
   InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override {
-    // TODO: Compute accurate cost after retiring the legacy cost model.
-    return 0;
-  }
+                              VPCostContext &Ctx) const override;
 
   VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
   VPValue *getIndex() const { return getOperand(1); }

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c3867024c34dc..badcd40daf46b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2797,6 +2797,86 @@ bool VPWidenIntOrFpInductionRecipe::isCanonical() const {
          getScalarType() == getRegion()->getCanonicalIVType();
 }
 
+InstructionCost VPDerivedIVRecipe::computeCost(ElementCount VF,
+                                               VPCostContext &Ctx) const {
+  // The cost model for this is modelled on expandVPDerivedIV in
+  // VPlanTransforms.cpp. In order to avoid overly pessimistic costs that can
+  // negatively affect vectorization it takes into account any expected
+  // simplifications that happen in simplifyRecipe.
+  switch (getInductionKind()) {
+  default:
+    // TODO: Compute cost for remaining kinds.
+    break;
+  case InductionDescriptor::IK_IntInduction: {
+    // There are currently no tests that expose a path where all lanes are
+    // used, so it's better to bail out for now.
+    if (!vputils::onlyFirstLaneUsed(this))
+      break;
+
+    // Start off by assuming we need both mul and add, then refine this.
+    bool NeedsMul = true, NeedsAdd = true, NeedsShl = false;
+
+    // If the start value is zero the add gets folded away.
+    if (auto *VPV = dyn_cast<VPIRValue>(getStartValue()))
+      if (auto *StartC = dyn_cast<ConstantInt>(VPV->getValue()))
+        NeedsAdd = !StartC->isZero();
+
+    // For some values of step the arithmetic changes:
+    //  1. A step of 1 requires no operation.
+    //  2. A step of -1 requires a negate.
+    //  3. A power-of-2 step will use a shl, instead of a mul.
+    Type *StepTy = getStepValue()->getScalarType();
+    InstructionCost Cost(0);
+    if (auto *VPV = dyn_cast<VPIRValue>(getStepValue())) {
+      if (auto *StepC = dyn_cast<ConstantInt>(VPV->getValue())) {
+        if (StepC->isOne())
+          NeedsMul = false;
+        else if (StepC->isMinusOne()) {
+          // This will most likely end up as a negate in simplifyRecipe, and
+          // the negate will be combined with the add to make a sub.
+          // NOTE: This is perhaps an invalid assumption that the cost of an
+          // 'add' is the same as a 'sub'.
+          NeedsMul = false;
+          NeedsAdd = true;
+        } else if (StepC->getValue().isPowerOf2()) {
+          // This will most likely end up as a shift-left in simplifyRecipe
+          NeedsMul = false;
+          NeedsShl = true;
+        }
+      }
+    }
+
+    // Add the cost of the conversion from index to step type if the index
+    // will be used.
+    Type *IndexTy = getIndex()->getScalarType();
+    unsigned StepTySize = StepTy->getScalarSizeInBits();
+    unsigned IndexTySize = IndexTy->getScalarSizeInBits();
+    if ((NeedsAdd || NeedsMul || NeedsShl) && StepTySize != IndexTySize) {
+      unsigned CastOpc =
+          StepTySize < IndexTySize ? Instruction::Trunc : Instruction::SExt;
+      Cost += Ctx.TTI.getCastInstrCost(
+          CastOpc, StepTy, IndexTy, TTI::CastContextHint::None, Ctx.CostKind);
+    }
+
+    if (NeedsMul)
+      Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, StepTy,
+                                             Ctx.CostKind);
+    if (NeedsShl)
+      Cost += Ctx.TTI.getArithmeticInstrCost(
+          Instruction::Shl, StepTy, Ctx.CostKind,
+          {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+          {TargetTransformInfo::OK_UniformConstantValue,
+           TargetTransformInfo::OP_None});
+    if (NeedsAdd)
+      Cost += Ctx.TTI.getArithmeticInstrCost(Instruction::Add, StepTy,
+                                             Ctx.CostKind);
+    return Cost;
+  }
+  }
+
+  return 0;
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPDerivedIVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
                                     VPSlotTracker &SlotTracker) const {

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index ef6fc9779c685..54dbddff5d9fd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -321,5 +321,7 @@ exit:
   ret void
 }
 
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !2, !3}
 !1 = !{!"llvm.loop.vectorize.width", i32 2}
+!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
index 01320424d431b..3fd329ede68ef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -12,10 +12,10 @@ define i64 @same_exit_block_pre_inc_use1_sve() #1 {
 ; CHECK: Calculating cost of work in exit block vector.early.exit
 ; CHECK-NEXT: Cost of 4 for VF vscale x 16: EMIT vp<{{.*}}> = first-active-lane ir<%cmp3>
 ; CHECK-NEXT: Cost of 0 for VF vscale x 16: EMIT vp<{{.*}}> = add
-; CHECK-NEXT: Cost of 0 for VF vscale x 16: vp<{{.*}}> = DERIVED-IV
+; CHECK-NEXT: Cost of 1 for VF vscale x 16: vp<{{.*}}> = DERIVED-IV
 ; CHECK-NEXT: Cost of 4 for VF vscale x 16: EMIT vp<{{.*}}> = first-active-lane ir<%cmp3>
 ; CHECK-NEXT: Cost of 0 for VF vscale x 16: EMIT vp<{{.*}}> = add
-; CHECK-NEXT: Cost of 0 for VF vscale x 16: vp<{{.*}}> = DERIVED-IV
+; CHECK-NEXT: Cost of 2 for VF vscale x 16: vp<{{.*}}> = DERIVED-IV
 ; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
 entry:
   %p1 = alloca [1024 x i8]
@@ -53,10 +53,10 @@ define i64 @same_exit_block_pre_inc_use1_nosve() {
 ; CHECK: Calculating cost of work in exit block vector.early.exit
 ; CHECK-NEXT: Cost of 48 for VF 16: EMIT vp<{{.*}}> = first-active-lane ir<%cmp3>
 ; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<{{.*}}> = add
-; CHECK-NEXT: Cost of 0 for VF 16: vp<{{.*}}> = DERIVED-IV
+; CHECK-NEXT: Cost of 1 for VF 16: vp<{{.*}}> = DERIVED-IV
 ; CHECK-NEXT: Cost of 48 for VF 16: EMIT vp<{{.*}}> = first-active-lane ir<%cmp3>
 ; CHECK-NEXT: Cost of 0 for VF 16: EMIT vp<{{.*}}> = add
-; CHECK-NEXT: Cost of 0 for VF 16: vp<{{.*}}> = DERIVED-IV
+; CHECK-NEXT: Cost of 2 for VF 16: vp<{{.*}}> = DERIVED-IV
 ; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
 entry:
   %p1 = alloca [1024 x i8]

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
index 42b2efaea2e89..99e0bfaec7ac6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
@@ -4,101 +4,73 @@
 define double @fp128_fmuladd_reduction(ptr %start0, ptr %start1, ptr %end0, ptr %end1, double %x, i64 %n) {
 ; CHECK-LABEL: define double @fp128_fmuladd_reduction(
 ; CHECK-SAME: ptr [[START0:%.*]], ptr [[START1:%.*]], ptr [[END0:%.*]], ptr [[END1:%.*]], double [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ITER_CHECK:.*]]:
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[N_VEC]], 4
-; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N_VEC]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP2]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi double [ [[X]], %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr fp128, ptr [[TMP1]], i64 2
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr fp128, ptr [[TMP1]], i64 4
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr fp128, ptr [[TMP1]], i64 6
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x fp128>, ptr [[TMP1]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <2 x fp128>, ptr [[TMP24]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x fp128>, ptr [[TMP4]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x fp128>, ptr [[TMP5]], align 16
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[TMP3]], i64 4
-; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr double, ptr [[TMP3]], i64 6
-; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP3]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <2 x double>, ptr [[TMP28]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <2 x double>, ptr [[TMP35]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <2 x double>, ptr [[TMP36]], align 16
-; CHECK-NEXT:    [[TMP10:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD]] to <2 x double>
-; CHECK-NEXT:    [[TMP11:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD3]] to <2 x double>
-; CHECK-NEXT:    [[TMP12:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD4]] to <2 x double>
-; CHECK-NEXT:    [[TMP13:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD5]] to <2 x double>
-; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[TMP10]], [[WIDE_LOAD6]]
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul <2 x double> [[TMP11]], [[WIDE_LOAD7]]
-; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[TMP12]], [[WIDE_LOAD8]]
-; CHECK-NEXT:    [[TMP17:%.*]] = fmul <2 x double> [[TMP13]], [[WIDE_LOAD9]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP14]])
-; CHECK-NEXT:    [[TMP19:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP18]], <2 x double> [[TMP15]])
-; CHECK-NEXT:    [[TMP20:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP19]], <2 x double> [[TMP16]])
-; CHECK-NEXT:    [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP20]], <2 x double> [[TMP17]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi double [ [[X]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP4]], 48
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP4]]
+; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP5]]
+; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP6]]
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP8]], 16
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP8]], 24
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP8]]
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP9]]
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP10]]
+; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load fp128, ptr [[NEXT_GEP]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = load fp128, ptr [[NEXT_GEP1]], align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = load fp128, ptr [[NEXT_GEP2]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = load fp128, ptr [[NEXT_GEP3]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = load double, ptr [[NEXT_GEP4]], align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = load double, ptr [[NEXT_GEP5]], align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = load double, ptr [[NEXT_GEP6]], align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = load double, ptr [[NEXT_GEP7]], align 16
+; CHECK-NEXT:    [[TMP20:%.*]] = fptrunc fp128 [[TMP12]] to double
+; CHECK-NEXT:    [[TMP21:%.*]] = fptrunc fp128 [[TMP13]] to double
+; CHECK-NEXT:    [[TMP22:%.*]] = fptrunc fp128 [[TMP14]] to double
+; CHECK-NEXT:    [[TMP23:%.*]] = fptrunc fp128 [[TMP15]] to double
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul double [[TMP20]], [[TMP16]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fmul double [[TMP21]], [[TMP17]]
+; CHECK-NEXT:    [[TMP26:%.*]] = fmul double [[TMP22]], [[TMP18]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fmul double [[TMP23]], [[TMP19]]
+; CHECK-NEXT:    [[TMP28:%.*]] = fadd double [[VEC_PHI]], [[TMP24]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fadd double [[TMP28]], [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = fadd double [[TMP29]], [[TMP26]]
+; CHECK-NEXT:    [[TMP31]] = fadd double [[TMP30]], [[TMP27]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
-; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
-; CHECK:       [[VEC_EPILOG_PH]]:
-; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP21]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[X]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[N_MOD_VF10:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT:    [[N_VEC11:%.*]] = sub i64 [[N]], [[N_MOD_VF10]]
-; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[N_VEC11]], 4
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP25]]
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[N_VEC11]], 3
-; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP8]]
-; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI13:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX12]], 4
-; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[START0]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[INDEX12]], 3
-; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP9]]
-; CHECK-NEXT:    [[WIDE_LOAD17:%.*]] = load <2 x fp128>, ptr [[NEXT_GEP14]], align 16
-; CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <2 x double>, ptr [[NEXT_GEP8]], align 16
-; CHECK-NEXT:    [[TMP31:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD17]] to <2 x double>
-; CHECK-NEXT:    [[TMP32:%.*]] = fmul <2 x double> [[TMP31]], [[WIDE_LOAD18]]
-; CHECK-NEXT:    [[TMP33]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI13]], <2 x double> [[TMP32]])
-; CHECK-NEXT:    [[INDEX_NEXT19]] = add nuw i64 [[INDEX12]], 2
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC11]]
-; CHECK-NEXT:    br i1 [[TMP34]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[N]], [[N_VEC11]]
-; CHECK-NEXT:    br i1 [[CMP_N20]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL21:%.*]] = phi ptr [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[NEXT_GEP3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START0]], %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL22:%.*]] = phi ptr [ [[NEXT_GEP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[NEXT_GEP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START1]], %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL23:%.*]] = phi i64 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX24:%.*]] = phi double [ [[TMP33]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[X]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[START1]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[X]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PTR0:%.*]] = phi ptr [ [[PTR0_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL21]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[PTR1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL22]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL23]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi double [ [[RED_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX24]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR0:%.*]] = phi ptr [ [[PTR0_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[PTR1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL9]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi double [ [[RED_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PTR0_NEXT]] = getelementptr i8, ptr [[PTR0]], i64 16
 ; CHECK-NEXT:    [[PTR1_NEXT]] = getelementptr i8, ptr [[PTR1]], i64 8
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load fp128, ptr [[PTR0]], align 16
@@ -107,9 +79,9 @@ define double @fp128_fmuladd_reduction(ptr %start0, ptr %start1, ptr %end0, ptr
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call double @llvm.fmuladd.f64(double [[TRUNC]], double [[LOAD1]], double [[RED]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[LCSSA:%.*]] = phi double [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[TMP33]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[LCSSA:%.*]] = phi double [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret double [[LCSSA]]
 ;
 entry:
@@ -138,7 +110,5 @@ exit:
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[PROF3]] = !{!"branch_weights", i32 2, i32 6}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
 ;.

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
index cb2b67faa2d46..fc4f233218a10 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always.ll
@@ -8,12 +8,12 @@ target triple = "aarch64"
 ; but that a scalable VF is still chosen (due to the `-scalable-vectorization=always` flag)
 define i32 @cost_prefers_fixed_width_vf_but_force_scalable_vf(ptr noalias %dst, ptr noalias %src, i64 %n) "target-cpu"="neoverse-n2" {
 ; CHECK-LABEL: 'cost_prefers_fixed_width_vf_but_force_scalable_vf'
-; CHECK:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
-; CHECK:  Cost for VF 4: 7 (Estimated cost per lane: 1.8)
-; CHECK:  Cost for VF 8: 9 (Estimated cost per lane: 1.1)
+; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6.0)
+; CHECK:  Cost for VF 4: 8 (Estimated cost per lane: 2.0)
+; CHECK:  Cost for VF 8: 10 (Estimated cost per lane: 1.2)
 ; CHECK:  Cost for VF vscale x 1: Invalid (Estimated cost per lane: Invalid)
-; CHECK:  Cost for VF vscale x 2: 8 (Estimated cost per lane: 4.0)
-; CHECK:  Cost for VF vscale x 4: 7 (Estimated cost per lane: 1.8)
+; CHECK:  Cost for VF vscale x 2: 9 (Estimated cost per lane: 4.5)
+; CHECK:  Cost for VF vscale x 4: 8 (Estimated cost per lane: 2.0)
 ; CHECK:  LV: Selecting VF: vscale x 4.
 ; CHECK:  VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={2}' {
 ;

diff  --git a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
index 987264482c8c6..a8016c98d8fdd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fully-unrolled-cost.ll
@@ -79,11 +79,11 @@ define i64 @test_two_ivs(ptr %a, ptr %b, i64 %start) #0 {
 ; CHECK-NEXT: Cost of 1 for VF 8: induction instruction   %j.iv.next = add nuw nsw i64 %j.iv, 1
 ; CHECK-NEXT: Cost of 0 for VF 8: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
 ; CHECK: Cost of 1 for VF 8: EMIT branch-on-count vp<%index.next>, vp<{{.+}}>
-; CHECK: Cost for VF 8: 16
+; CHECK: Cost for VF 8: 17
 ; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
 ; CHECK-NEXT: Cost of 0 for VF 16: induction instruction   %j.iv = phi i64 [ %start, %entry ], [ %j.iv.next, %for.body ]
 ; CHECK: Cost of 1 for VF 16: EXPRESSION vp<%11> = ir<%sum> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i64), (ir<%0> zext to i64))
-; CHECK: Cost for VF 16: 3
+; CHECK: Cost for VF 16: 4
 ; CHECK: LV: Selecting VF: 16
 entry:
   br label %for.body

diff  --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
index 193bd60a9dfe2..67db96c569a21 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
@@ -172,12 +172,12 @@ exit:
 define void @spills_profitable(ptr %in1, ptr %in2, ptr %out, i32 %n, i32 %m) {
 ; CHECK-LABEL: LV: Checking a loop in 'spills_profitable'
 ; CHECK: LV: Scalar loop costs: 54
-; CHECK-NOPRESSURE: Cost for VF 2: 1530 (Estimated cost per lane: 765.0)
-; CHECK-NOPRESSURE: Cost for VF 4: 38 (Estimated cost per lane: 9.5)
+; CHECK-NOPRESSURE: Cost for VF 2: 1535 (Estimated cost per lane: 767.5)
+; CHECK-NOPRESSURE: Cost for VF 4: 43 (Estimated cost per lane: 10.8)
 ; CHECK-PRESSURE: LV(REG): Cost of 4 from 2 spills of Generic::ScalarRC
-; CHECK-PRESSURE-NEXT: Cost for VF 2: 1534 (Estimated cost per lane: 767.0)
+; CHECK-PRESSURE-NEXT: Cost for VF 2: 1539 (Estimated cost per lane: 769.5)
 ; CHECK-PRESSURE: LV(REG): Cost of 6 from 3 spills of Generic::VectorRC
-; CHECK-PRESSURE-NEXT: Cost for VF 4: 44 (Estimated cost per lane: 11.0)
+; CHECK-PRESSURE-NEXT: Cost for VF 4: 49 (Estimated cost per lane: 12.2)
 ; CHECK: LV: Selecting VF: 4
 entry:
   %cmp = icmp eq i32 %n, 0

diff  --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index efae5f75fc625..c09ce83d02266 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -179,10 +179,12 @@ define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
 ; RV64-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 88
 ; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; RV64-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP1]]
+; RV64-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 3)
+; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[UMAX]]
 ; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
-; RV64-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1
+; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; RV64-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
 ; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 132, [[TMP3]]
 ; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
 ; RV64-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 112

diff  --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
index 4271ee80d366c..df53040407d28 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
@@ -282,45 +282,58 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ;
 ; NO-VP-LABEL: @multiple_reverse_vector_pointer(
 ; NO-VP-NEXT:  entry:
-; NO-VP-NEXT:    br label [[VECTOR_PH:%.*]]
+; NO-VP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP5]], 4
+; NO-VP-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 32)
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[UMAX]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP4:%.*]] = sub i64 1024, [[N_VEC]]
 ; NO-VP-NEXT:    br label [[LOOP:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
 ; NO-VP-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1024, [[INDEX]]
 ; NO-VP-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; NO-VP-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -15
-; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; NO-VP-NEXT:    [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; NO-VP-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[B:%.*]], <16 x i8> [[REVERSE]]
-; NO-VP-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 [[TMP3]], <16 x i1> splat (i1 true), <16 x i8> poison)
-; NO-VP-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[OFFSET_IDX]]
-; NO-VP-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP4]], i64 -15
-; NO-VP-NEXT:    [[REVERSE1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_GATHER]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; NO-VP-NEXT:    store <16 x i8> [[REVERSE1]], ptr [[TMP6]], align 1
+; NO-VP-NEXT:    [[TMP11:%.*]] = sub nuw nsw i64 [[TMP3]], 1
+; NO-VP-NEXT:    [[TMP8:%.*]] = sub i64 0, [[TMP11]]
+; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[TMP8]]
+; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1
+; NO-VP-NEXT:    [[REVERSE:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[B:%.*]], <vscale x 16 x i8> [[REVERSE]]
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP10]], <vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> poison)
 ; NO-VP-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 [[OFFSET_IDX]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP7]], i64 -15
-; NO-VP-NEXT:    store <16 x i8> [[REVERSE1]], ptr [[TMP9]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[TMP8]]
+; NO-VP-NEXT:    [[REVERSE1:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]])
+; NO-VP-NEXT:    store <vscale x 16 x i8> [[REVERSE1]], ptr [[TMP12]], align 1
+; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[D1:%.*]], i64 [[OFFSET_IDX]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP8]]
+; NO-VP-NEXT:    store <vscale x 16 x i8> [[REVERSE1]], ptr [[TMP14]], align 1
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; NO-VP-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       middle.block:
-; NO-VP-NEXT:    br label [[SCALAR_PH:%.*]]
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 1024, [[ENTRY:%.*]] ]
 ; NO-VP-NEXT:    br label [[LOOP1:%.*]]
 ; NO-VP:       loop:
-; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP1]] ]
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP1]] ]
 ; NO-VP-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    [[X:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; NO-VP-NEXT:    [[GEP_B:%.*]] = getelementptr i8, ptr [[B]], i8 [[X]]
 ; NO-VP-NEXT:    [[Y:%.*]] = load i8, ptr [[GEP_B]], align 1
-; NO-VP-NEXT:    [[GEP_C:%.*]] = getelementptr i8, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT:    [[GEP_C:%.*]] = getelementptr i8, ptr [[D]], i64 [[IV]]
 ; NO-VP-NEXT:    store i8 [[Y]], ptr [[GEP_C]], align 1
-; NO-VP-NEXT:    [[GEP_D:%.*]] = getelementptr i8, ptr [[D]], i64 [[IV]]
+; NO-VP-NEXT:    [[GEP_D:%.*]] = getelementptr i8, ptr [[D1]], i64 [[IV]]
 ; NO-VP-NEXT:    store i8 [[Y]], ptr [[GEP_D]], align 1
 ; NO-VP-NEXT:    [[IV_NEXT]] = add i64 [[IV]], -1
 ; NO-VP-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[IV]], 0
-; NO-VP-NEXT:    br i1 [[CMP_NOT]], label [[EXIT:%.*]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; NO-VP-NEXT:    br i1 [[CMP_NOT]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
 ; NO-VP:       exit:
 ; NO-VP-NEXT:    ret void
 ;

diff  --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
index 2bdff5e37f95f..6a0a3410a9585 100644
--- a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
@@ -1621,21 +1621,21 @@ define hidden void @four_bytes_into_four_ints_vary_op(ptr noalias nocapture noun
 ; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%11> to index 0
 ; CHECK-NEXT:   store ir<%13> to index 1
-; CHECK: Cost for VF 4: 35 (Estimated cost per lane: 8.
+; CHECK: Cost for VF 4: 37 (Estimated cost per lane: 9.
 ; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%10>
 ; CHECK-NEXT:   ir<%11> = load from index 0
 ; CHECK-NEXT:   ir<%13> = load from index 1
 ; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%11> to index 0
 ; CHECK-NEXT:   store ir<%13> to index 1
-; CHECK: Cost for VF 8: 39 (Estimated cost per lane: 4.
+; CHECK: Cost for VF 8: 41 (Estimated cost per lane: 5.
 ; CHECK: Cost of 68 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%10>
 ; CHECK-NEXT:   ir<%11> = load from index 0
 ; CHECK-NEXT:   ir<%13> = load from index 1
 ; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%11> to index 0
 ; CHECK-NEXT:   store ir<%13> to index 1
-; CHECK: Cost for VF 16: 80 (Estimated cost per lane: 5.
+; CHECK: Cost for VF 16: 82 (Estimated cost per lane: 5.
 ; CHECK: LV: Selecting VF: 8.
 define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
   %5 = icmp sgt i32 %3, 0
@@ -1668,7 +1668,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
 ; CHECK: Cost of 6 for VF 2: REPLICATE ir<%17> = load ir<%16> (!alias.scope {{.*}})
 ; CHECK: Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19> (!alias.scope {{.*}})
 ; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%48>, ir<%49>
-; CHECK: Cost for VF 2: 78 (Estimated cost per lane: 39.
+; CHECK: Cost for VF 2: 80 (Estimated cost per lane: 40.
 ; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%14> = load from index 0
 ; CHECK-NEXT:   ir<%32> = load from index 1
@@ -1678,7 +1678,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
 ; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%30> to index 0
 ; CHECK-NEXT:   store ir<%48> to index 1
-; CHECK: Cost for VF 4: 73 (Estimated cost per lane: 18.
+; CHECK: Cost for VF 4: 75 (Estimated cost per lane: 18.
 ; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%14> = load from index 0
 ; CHECK-NEXT:   ir<%32> = load from index 1
@@ -1688,7 +1688,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
 ; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%30> to index 0
 ; CHECK-NEXT:   store ir<%48> to index 1
-; CHECK: Cost for VF 8: 89 (Estimated cost per lane: 11.
+; CHECK: Cost for VF 8: 91 (Estimated cost per lane: 11.
 ; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%14> = load from index 0
 ; CHECK-NEXT:   ir<%32> = load from index 1
@@ -1698,7 +1698,7 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
 ; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%30> to index 0
 ; CHECK-NEXT:   store ir<%48> to index 1
-; CHECK: Cost for VF 16: 322 (Estimated cost per lane: 20.
+; CHECK: Cost for VF 16: 324 (Estimated cost per lane: 20.
 ; CHECK: LV: Selecting VF: 8.
 define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
   %5 = icmp sgt i32 %3, 0
@@ -1768,7 +1768,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
 ; CHECK: Cost of 6 for VF 2: REPLICATE ir<%13> = load ir<%12> (!alias.scope {{.*}})
 ; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%18>, vp<%next.gep>.1 (!alias.scope {{.*}}, !noalias {{.*}})
 ; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%28>, ir<%29>
-; CHECK: Cost for VF 2: 50 (Estimated cost per lane: 25.
+; CHECK: Cost for VF 2: 52 (Estimated cost per lane: 26.
 ; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%10> = load from index 0
 ; CHECK-NEXT:   ir<%20> = load from index 1
@@ -1777,7 +1777,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
 ; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%18> to index 0
 ; CHECK-NEXT:   store ir<%28> to index 1
-; CHECK: Cost for VF 4: 47 (Estimated cost per lane: 11.
+; CHECK: Cost for VF 4: 49 (Estimated cost per lane: 12.
 ; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%10> = load from index 0
 ; CHECK-NEXT:   ir<%20> = load from index 1
@@ -1786,7 +1786,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
 ; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%18> to index 0
 ; CHECK-NEXT:   store ir<%28> to index 1
-; CHECK: Cost for VF 8: 55 (Estimated cost per lane: 6.
+; CHECK: Cost for VF 8: 57 (Estimated cost per lane: 7.
 ; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
 ; CHECK-NEXT:   ir<%10> = load from index 0
 ; CHECK-NEXT:   ir<%20> = load from index 1
@@ -1795,7 +1795,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
 ; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
 ; CHECK-NEXT:   store ir<%18> to index 0
 ; CHECK-NEXT:   store ir<%28> to index 1
-; CHECK: Cost for VF 16: 174 (Estimated cost per lane: 10.
+; CHECK: Cost for VF 16: 176 (Estimated cost per lane: 11.
 ; CHECK: LV: Selecting VF: 8.
 define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0, i32 noundef %1, ptr nocapture noundef writeonly %2, i32 noundef %3) {
   %5 = icmp sgt i32 %3, 0

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index c71139e93caab..421ce3c18d95b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -25,13 +25,13 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
 ; CHECK:  Cost of 0 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
 ; CHECK:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.src>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.src>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
 ; CHECK:  Cost of 1 for VF 2: WIDEN ir<%iv.4> = add ir<%iv>, ir<4>
 ; CHECK:  Cost of 1 for VF 2: WIDEN ir<%c> = icmp ule ir<%l>, ir<128>
 ; CHECK:  Cost of 1 for VF 2: EMIT ir<%or> = add ir<%iv.4>, ir<1>
 ; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%g.dst>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
@@ -49,13 +49,13 @@ define void @wide_or_replaced_with_add_vpinstruction(ptr %src, ptr noalias %dst)
 ; CHECK:  Cost of 0 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.src>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.src>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
 ; CHECK:  Cost of 1 for VF 4: WIDEN ir<%iv.4> = add ir<%iv>, ir<4>
 ; CHECK:  Cost of 1 for VF 4: WIDEN ir<%c> = icmp ule ir<%l>, ir<128>
 ; CHECK:  Cost of 1 for VF 4: EMIT ir<%or> = add ir<%iv.4>, ir<1>
 ; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr ir<%dst>, ir<%or>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer ir<%g.dst>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%iv.4>, ir<%c>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -112,11 +112,11 @@ define void @test_vpinstruction_freeze_cost(ptr %src, ptr noalias %dst) {
 ; CHECK:  Cost of 0 for VF 2: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
 ; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.src>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.src>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
 ; CHECK:  Cost of 0 for VF 2: WIDEN ir<%fr> = freeze ir<%l>
 ; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%g.dst>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%fr>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
@@ -136,11 +136,11 @@ define void @test_vpinstruction_freeze_cost(ptr %src, ptr noalias %dst) {
 ; CHECK:  Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:  Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.src>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.src>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
 ; CHECK:  Cost of 0 for VF 4: WIDEN ir<%fr> = freeze ir<%l>
 ; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds ir<%g.dst>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%fr>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -195,10 +195,10 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
 ; CHECK:  LV: Found an estimated cost of 0 for VF 1 For instruction: br i1 %ec, label %exit, label %loop.header
 ; CHECK:  Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
 ; CHECK:  Cost of 0 for VF 2: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
-; CHECK:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
+; CHECK:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
 ; CHECK:  Cost of 0 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq ir<%l>, ir<-12>
 ; CHECK:  Cost of 1 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq ir<%l>, ir<13>
@@ -206,13 +206,13 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP11]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = not vp<[[VP13]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP15:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP15:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP16:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP16:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP16]]>, ir<0>, vp<[[VP10]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP17:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP17:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP17]]>, ir<42>, vp<[[VP9]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP18:%[0-9]+]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP18:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP18]]>, ir<2>, vp<[[VP14]]>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1:%[0-9]+]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
@@ -226,10 +226,10 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
 ; CHECK:  Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
 ; CHECK:  Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
 ; CHECK:  Cost of 0 for VF 4: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
-; CHECK:  Cost of 0 for VF 4: vp<[[VP6]]> = DERIVED-IV ir<0> + vp<[[VP5]]> * ir<8>
+; CHECK:  Cost of 1 for VF 4: vp<[[VP6]]> = DERIVED-IV ir<0> + vp<[[VP5]]> * ir<8>
 ; CHECK:  Cost of 0 for VF 4: vp<[[VP7]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP8]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP8]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP8]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT vp<[[VP9]]> = icmp eq ir<%l>, ir<-12>
 ; CHECK:  Cost of 1 for VF 4: EMIT vp<[[VP10]]> = icmp eq ir<%l>, ir<13>
@@ -237,13 +237,13 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<[[VP12]]> = or vp<[[VP9]]>, vp<[[VP10]]>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<[[VP13]]> = or vp<[[VP12]]>, vp<[[VP11]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT vp<[[VP14]]> = not vp<[[VP13]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP15]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP15]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP15]]>, ir<1>, vp<[[VP11]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP16]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP16]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP16]]>, ir<0>, vp<[[VP10]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP17]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP17]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP17]]>, ir<42>, vp<[[VP9]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP18]]> = vector-pointer vp<%next.gep>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP18]]> = vector-pointer vp<%next.gep>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP18]]>, ir<2>, vp<[[VP14]]>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -308,7 +308,7 @@ define void @test_vpinstruction_extractvalue_cost(ptr noalias %dst, {i64, i64} %
 ; CHECK:  Cost of 0 for VF 2: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
 ; CHECK:  Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.dst>
+; CHECK:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 2: WIDEN store vp<[[VP5]]>, ir<%add>
 ; CHECK:  Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
 ; CHECK:  Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
@@ -331,7 +331,7 @@ define void @test_vpinstruction_extractvalue_cost(ptr noalias %dst, {i64, i64} %
 ; CHECK:  Cost of 0 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 ; CHECK:  Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
 ; CHECK:  Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
-; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.dst>
+; CHECK:  Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.dst>, ir<1>
 ; CHECK:  Cost of 1 for VF 4: WIDEN store vp<[[VP5]]>, ir<%add>
 ; CHECK:  Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
 ; CHECK:  Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
index e62552f936697..a8e32ca9a0c8c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
@@ -12,7 +12,7 @@ define void @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) {
 ; CHECK:       [[ITER_CHECK]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[N]], -3
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 8
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], 32
@@ -36,27 +36,27 @@ define void @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], [[DOT_CRIT_EDGE_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 3, [[N_VEC3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[BC_RESUME_VAL]] to i8
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP9]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP9]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND5:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND5:%.*]] = phi <8 x i8> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i64 3, [[INDEX4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT:    store <4 x i8> [[VEC_IND5]], ptr [[TMP11]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT7]] = add <4 x i8> [[VEC_IND5]], splat (i8 4)
+; CHECK-NEXT:    store <8 x i8> [[VEC_IND5]], ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 8
+; CHECK-NEXT:    [[VEC_IND_NEXT7]] = add <8 x i8> [[VEC_IND5]], splat (i8 8)
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -113,31 +113,16 @@ define void @conversion_cost2(i32 %n, ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 9, i64 10>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    [[STEP_ADD_2:%.*]] = add <2 x i64> [[STEP_ADD]], splat (i64 2)
-; CHECK-NEXT:    [[STEP_ADD_3:%.*]] = add <2 x i64> [[STEP_ADD_2]], splat (i64 2)
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 9, [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <2 x i64> [[STEP_ADD]], splat (i64 3)
-; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <2 x i64> [[STEP_ADD_2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <2 x i64> [[STEP_ADD_3]], splat (i64 3)
-; CHECK-NEXT:    [[TMP10:%.*]] = sitofp <2 x i64> [[TMP6]] to <2 x float>
-; CHECK-NEXT:    [[TMP11:%.*]] = sitofp <2 x i64> [[TMP7]] to <2 x float>
-; CHECK-NEXT:    [[TMP12:%.*]] = sitofp <2 x i64> [[TMP8]] to <2 x float>
-; CHECK-NEXT:    [[TMP13:%.*]] = sitofp <2 x i64> [[TMP9]] to <2 x float>
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 2
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 4
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 6
-; CHECK-NEXT:    store <2 x float> [[TMP10]], ptr [[TMP14]], align 4
-; CHECK-NEXT:    store <2 x float> [[TMP11]], ptr [[TMP15]], align 4
-; CHECK-NEXT:    store <2 x float> [[TMP12]], ptr [[TMP16]], align 4
-; CHECK-NEXT:    store <2 x float> [[TMP13]], ptr [[TMP17]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <8 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <8 x i64> [[TMP6]] to <8 x float>
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT:    store <8 x float> [[TMP7]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD_3]], splat (i64 2)
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], [[DOT_CRIT_EDGE_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 0cbb8a8cc4bb4..47215a17934be 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -79,19 +79,19 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], 16
+; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], 32
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 32
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[N_VEC]], 5
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP120:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP121:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP122:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP123:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP232:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP233:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP234:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP235:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[INDEX]], 5
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 32
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], 64
@@ -108,118 +108,230 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP4]], 416
 ; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP4]], 448
 ; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP4]], 480
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP36:%.*]] = load float, ptr [[TMP20]], align 4
-; CHECK-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP21]], align 4
-; CHECK-NEXT:    [[TMP38:%.*]] = load float, ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP23]], align 4
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP36]], i32 0
-; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP37]], i32 1
-; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP38]], i32 2
-; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP39]], i32 3
-; CHECK-NEXT:    [[TMP44:%.*]] = load float, ptr [[TMP24]], align 4
-; CHECK-NEXT:    [[TMP45:%.*]] = load float, ptr [[TMP25]], align 4
-; CHECK-NEXT:    [[TMP46:%.*]] = load float, ptr [[TMP26]], align 4
-; CHECK-NEXT:    [[TMP47:%.*]] = load float, ptr [[TMP27]], align 4
-; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <4 x float> poison, float [[TMP44]], i32 0
-; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <4 x float> [[TMP48]], float [[TMP45]], i32 1
-; CHECK-NEXT:    [[TMP50:%.*]] = insertelement <4 x float> [[TMP49]], float [[TMP46]], i32 2
-; CHECK-NEXT:    [[TMP51:%.*]] = insertelement <4 x float> [[TMP50]], float [[TMP47]], i32 3
-; CHECK-NEXT:    [[TMP52:%.*]] = load float, ptr [[TMP28]], align 4
-; CHECK-NEXT:    [[TMP53:%.*]] = load float, ptr [[TMP29]], align 4
-; CHECK-NEXT:    [[TMP54:%.*]] = load float, ptr [[TMP30]], align 4
-; CHECK-NEXT:    [[TMP55:%.*]] = load float, ptr [[TMP31]], align 4
-; CHECK-NEXT:    [[TMP56:%.*]] = insertelement <4 x float> poison, float [[TMP52]], i32 0
-; CHECK-NEXT:    [[TMP57:%.*]] = insertelement <4 x float> [[TMP56]], float [[TMP53]], i32 1
-; CHECK-NEXT:    [[TMP58:%.*]] = insertelement <4 x float> [[TMP57]], float [[TMP54]], i32 2
-; CHECK-NEXT:    [[TMP59:%.*]] = insertelement <4 x float> [[TMP58]], float [[TMP55]], i32 3
-; CHECK-NEXT:    [[TMP60:%.*]] = load float, ptr [[TMP32]], align 4
-; CHECK-NEXT:    [[TMP61:%.*]] = load float, ptr [[TMP33]], align 4
-; CHECK-NEXT:    [[TMP62:%.*]] = load float, ptr [[TMP34]], align 4
-; CHECK-NEXT:    [[TMP63:%.*]] = load float, ptr [[TMP35]], align 4
-; CHECK-NEXT:    [[TMP64:%.*]] = insertelement <4 x float> poison, float [[TMP60]], i32 0
-; CHECK-NEXT:    [[TMP65:%.*]] = insertelement <4 x float> [[TMP64]], float [[TMP61]], i32 1
-; CHECK-NEXT:    [[TMP66:%.*]] = insertelement <4 x float> [[TMP65]], float [[TMP62]], i32 2
-; CHECK-NEXT:    [[TMP67:%.*]] = insertelement <4 x float> [[TMP66]], float [[TMP63]], i32 3
-; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP76:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP81:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP84:%.*]] = load float, ptr [[TMP68]], align 4
-; CHECK-NEXT:    [[TMP85:%.*]] = load float, ptr [[TMP69]], align 4
-; CHECK-NEXT:    [[TMP86:%.*]] = load float, ptr [[TMP70]], align 4
-; CHECK-NEXT:    [[TMP87:%.*]] = load float, ptr [[TMP71]], align 4
-; CHECK-NEXT:    [[TMP88:%.*]] = insertelement <4 x float> poison, float [[TMP84]], i32 0
-; CHECK-NEXT:    [[TMP89:%.*]] = insertelement <4 x float> [[TMP88]], float [[TMP85]], i32 1
-; CHECK-NEXT:    [[TMP90:%.*]] = insertelement <4 x float> [[TMP89]], float [[TMP86]], i32 2
-; CHECK-NEXT:    [[TMP91:%.*]] = insertelement <4 x float> [[TMP90]], float [[TMP87]], i32 3
-; CHECK-NEXT:    [[TMP92:%.*]] = load float, ptr [[TMP72]], align 4
-; CHECK-NEXT:    [[TMP93:%.*]] = load float, ptr [[TMP73]], align 4
-; CHECK-NEXT:    [[TMP94:%.*]] = load float, ptr [[TMP74]], align 4
-; CHECK-NEXT:    [[TMP95:%.*]] = load float, ptr [[TMP75]], align 4
-; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x float> poison, float [[TMP92]], i32 0
-; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <4 x float> [[TMP96]], float [[TMP93]], i32 1
-; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <4 x float> [[TMP97]], float [[TMP94]], i32 2
-; CHECK-NEXT:    [[TMP99:%.*]] = insertelement <4 x float> [[TMP98]], float [[TMP95]], i32 3
-; CHECK-NEXT:    [[TMP100:%.*]] = load float, ptr [[TMP76]], align 4
-; CHECK-NEXT:    [[TMP101:%.*]] = load float, ptr [[TMP77]], align 4
-; CHECK-NEXT:    [[TMP102:%.*]] = load float, ptr [[TMP78]], align 4
-; CHECK-NEXT:    [[TMP103:%.*]] = load float, ptr [[TMP79]], align 4
-; CHECK-NEXT:    [[TMP104:%.*]] = insertelement <4 x float> poison, float [[TMP100]], i32 0
-; CHECK-NEXT:    [[TMP105:%.*]] = insertelement <4 x float> [[TMP104]], float [[TMP101]], i32 1
-; CHECK-NEXT:    [[TMP106:%.*]] = insertelement <4 x float> [[TMP105]], float [[TMP102]], i32 2
-; CHECK-NEXT:    [[TMP107:%.*]] = insertelement <4 x float> [[TMP106]], float [[TMP103]], i32 3
-; CHECK-NEXT:    [[TMP108:%.*]] = load float, ptr [[TMP80]], align 4
-; CHECK-NEXT:    [[TMP109:%.*]] = load float, ptr [[TMP81]], align 4
-; CHECK-NEXT:    [[TMP110:%.*]] = load float, ptr [[TMP82]], align 4
-; CHECK-NEXT:    [[TMP111:%.*]] = load float, ptr [[TMP83]], align 4
-; CHECK-NEXT:    [[TMP112:%.*]] = insertelement <4 x float> poison, float [[TMP108]], i32 0
-; CHECK-NEXT:    [[TMP113:%.*]] = insertelement <4 x float> [[TMP112]], float [[TMP109]], i32 1
-; CHECK-NEXT:    [[TMP114:%.*]] = insertelement <4 x float> [[TMP113]], float [[TMP110]], i32 2
-; CHECK-NEXT:    [[TMP115:%.*]] = insertelement <4 x float> [[TMP114]], float [[TMP111]], i32 3
-; CHECK-NEXT:    [[TMP116:%.*]] = fadd fast <4 x float> [[TMP43]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP117:%.*]] = fadd fast <4 x float> [[TMP51]], [[VEC_PHI2]]
-; CHECK-NEXT:    [[TMP118:%.*]] = fadd fast <4 x float> [[TMP59]], [[VEC_PHI3]]
-; CHECK-NEXT:    [[TMP119:%.*]] = fadd fast <4 x float> [[TMP67]], [[VEC_PHI4]]
-; CHECK-NEXT:    [[TMP120]] = fadd fast <4 x float> [[TMP116]], [[TMP91]]
-; CHECK-NEXT:    [[TMP121]] = fadd fast <4 x float> [[TMP117]], [[TMP99]]
-; CHECK-NEXT:    [[TMP122]] = fadd fast <4 x float> [[TMP118]], [[TMP107]]
-; CHECK-NEXT:    [[TMP123]] = fadd fast <4 x float> [[TMP119]], [[TMP115]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP124:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP124]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[TMP4]], 512
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP4]], 544
+; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP4]], 576
+; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP4]], 608
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP4]], 640
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP4]], 672
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP4]], 704
+; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[TMP4]], 736
+; CHECK-NEXT:    [[TMP28:%.*]] = add i64 [[TMP4]], 768
+; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[TMP4]], 800
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP4]], 832
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP4]], 864
+; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP4]], 896
+; CHECK-NEXT:    [[TMP33:%.*]] = add i64 [[TMP4]], 928
+; CHECK-NEXT:    [[TMP34:%.*]] = add i64 [[TMP4]], 960
+; CHECK-NEXT:    [[TMP35:%.*]] = add i64 [[TMP4]], 992
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP26]]
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP28]]
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP29]]
+; CHECK-NEXT:    [[TMP62:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]]
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP34]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP68:%.*]] = load float, ptr [[TMP36]], align 4
+; CHECK-NEXT:    [[TMP69:%.*]] = load float, ptr [[TMP37]], align 4
+; CHECK-NEXT:    [[TMP70:%.*]] = load float, ptr [[TMP38]], align 4
+; CHECK-NEXT:    [[TMP71:%.*]] = load float, ptr [[TMP39]], align 4
+; CHECK-NEXT:    [[TMP72:%.*]] = load float, ptr [[TMP40]], align 4
+; CHECK-NEXT:    [[TMP73:%.*]] = load float, ptr [[TMP41]], align 4
+; CHECK-NEXT:    [[TMP74:%.*]] = load float, ptr [[TMP42]], align 4
+; CHECK-NEXT:    [[TMP75:%.*]] = load float, ptr [[TMP43]], align 4
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <8 x float> poison, float [[TMP68]], i32 0
+; CHECK-NEXT:    [[TMP77:%.*]] = insertelement <8 x float> [[TMP76]], float [[TMP69]], i32 1
+; CHECK-NEXT:    [[TMP78:%.*]] = insertelement <8 x float> [[TMP77]], float [[TMP70]], i32 2
+; CHECK-NEXT:    [[TMP79:%.*]] = insertelement <8 x float> [[TMP78]], float [[TMP71]], i32 3
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <8 x float> [[TMP79]], float [[TMP72]], i32 4
+; CHECK-NEXT:    [[TMP81:%.*]] = insertelement <8 x float> [[TMP80]], float [[TMP73]], i32 5
+; CHECK-NEXT:    [[TMP82:%.*]] = insertelement <8 x float> [[TMP81]], float [[TMP74]], i32 6
+; CHECK-NEXT:    [[TMP83:%.*]] = insertelement <8 x float> [[TMP82]], float [[TMP75]], i32 7
+; CHECK-NEXT:    [[TMP84:%.*]] = load float, ptr [[TMP44]], align 4
+; CHECK-NEXT:    [[TMP85:%.*]] = load float, ptr [[TMP45]], align 4
+; CHECK-NEXT:    [[TMP86:%.*]] = load float, ptr [[TMP46]], align 4
+; CHECK-NEXT:    [[TMP87:%.*]] = load float, ptr [[TMP47]], align 4
+; CHECK-NEXT:    [[TMP88:%.*]] = load float, ptr [[TMP48]], align 4
+; CHECK-NEXT:    [[TMP89:%.*]] = load float, ptr [[TMP49]], align 4
+; CHECK-NEXT:    [[TMP90:%.*]] = load float, ptr [[TMP50]], align 4
+; CHECK-NEXT:    [[TMP91:%.*]] = load float, ptr [[TMP51]], align 4
+; CHECK-NEXT:    [[TMP92:%.*]] = insertelement <8 x float> poison, float [[TMP84]], i32 0
+; CHECK-NEXT:    [[TMP93:%.*]] = insertelement <8 x float> [[TMP92]], float [[TMP85]], i32 1
+; CHECK-NEXT:    [[TMP94:%.*]] = insertelement <8 x float> [[TMP93]], float [[TMP86]], i32 2
+; CHECK-NEXT:    [[TMP95:%.*]] = insertelement <8 x float> [[TMP94]], float [[TMP87]], i32 3
+; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <8 x float> [[TMP95]], float [[TMP88]], i32 4
+; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <8 x float> [[TMP96]], float [[TMP89]], i32 5
+; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <8 x float> [[TMP97]], float [[TMP90]], i32 6
+; CHECK-NEXT:    [[TMP99:%.*]] = insertelement <8 x float> [[TMP98]], float [[TMP91]], i32 7
+; CHECK-NEXT:    [[TMP100:%.*]] = load float, ptr [[TMP52]], align 4
+; CHECK-NEXT:    [[TMP101:%.*]] = load float, ptr [[TMP53]], align 4
+; CHECK-NEXT:    [[TMP102:%.*]] = load float, ptr [[TMP54]], align 4
+; CHECK-NEXT:    [[TMP103:%.*]] = load float, ptr [[TMP55]], align 4
+; CHECK-NEXT:    [[TMP104:%.*]] = load float, ptr [[TMP56]], align 4
+; CHECK-NEXT:    [[TMP105:%.*]] = load float, ptr [[TMP57]], align 4
+; CHECK-NEXT:    [[TMP106:%.*]] = load float, ptr [[TMP58]], align 4
+; CHECK-NEXT:    [[TMP107:%.*]] = load float, ptr [[TMP59]], align 4
+; CHECK-NEXT:    [[TMP108:%.*]] = insertelement <8 x float> poison, float [[TMP100]], i32 0
+; CHECK-NEXT:    [[TMP109:%.*]] = insertelement <8 x float> [[TMP108]], float [[TMP101]], i32 1
+; CHECK-NEXT:    [[TMP110:%.*]] = insertelement <8 x float> [[TMP109]], float [[TMP102]], i32 2
+; CHECK-NEXT:    [[TMP111:%.*]] = insertelement <8 x float> [[TMP110]], float [[TMP103]], i32 3
+; CHECK-NEXT:    [[TMP112:%.*]] = insertelement <8 x float> [[TMP111]], float [[TMP104]], i32 4
+; CHECK-NEXT:    [[TMP113:%.*]] = insertelement <8 x float> [[TMP112]], float [[TMP105]], i32 5
+; CHECK-NEXT:    [[TMP114:%.*]] = insertelement <8 x float> [[TMP113]], float [[TMP106]], i32 6
+; CHECK-NEXT:    [[TMP115:%.*]] = insertelement <8 x float> [[TMP114]], float [[TMP107]], i32 7
+; CHECK-NEXT:    [[TMP116:%.*]] = load float, ptr [[TMP60]], align 4
+; CHECK-NEXT:    [[TMP117:%.*]] = load float, ptr [[TMP61]], align 4
+; CHECK-NEXT:    [[TMP118:%.*]] = load float, ptr [[TMP62]], align 4
+; CHECK-NEXT:    [[TMP119:%.*]] = load float, ptr [[TMP63]], align 4
+; CHECK-NEXT:    [[TMP120:%.*]] = load float, ptr [[TMP64]], align 4
+; CHECK-NEXT:    [[TMP121:%.*]] = load float, ptr [[TMP65]], align 4
+; CHECK-NEXT:    [[TMP122:%.*]] = load float, ptr [[TMP66]], align 4
+; CHECK-NEXT:    [[TMP123:%.*]] = load float, ptr [[TMP67]], align 4
+; CHECK-NEXT:    [[TMP124:%.*]] = insertelement <8 x float> poison, float [[TMP116]], i32 0
+; CHECK-NEXT:    [[TMP125:%.*]] = insertelement <8 x float> [[TMP124]], float [[TMP117]], i32 1
+; CHECK-NEXT:    [[TMP126:%.*]] = insertelement <8 x float> [[TMP125]], float [[TMP118]], i32 2
+; CHECK-NEXT:    [[TMP127:%.*]] = insertelement <8 x float> [[TMP126]], float [[TMP119]], i32 3
+; CHECK-NEXT:    [[TMP128:%.*]] = insertelement <8 x float> [[TMP127]], float [[TMP120]], i32 4
+; CHECK-NEXT:    [[TMP129:%.*]] = insertelement <8 x float> [[TMP128]], float [[TMP121]], i32 5
+; CHECK-NEXT:    [[TMP130:%.*]] = insertelement <8 x float> [[TMP129]], float [[TMP122]], i32 6
+; CHECK-NEXT:    [[TMP131:%.*]] = insertelement <8 x float> [[TMP130]], float [[TMP123]], i32 7
+; CHECK-NEXT:    [[TMP132:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP133:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP134:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP136:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP137:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP138:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP139:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP140:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP141:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP142:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP143:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP144:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP145:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP146:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP147:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP148:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP149:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP150:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP151:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP152:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP153:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP154:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP26]]
+; CHECK-NEXT:    [[TMP155:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP156:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP28]]
+; CHECK-NEXT:    [[TMP157:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP29]]
+; CHECK-NEXT:    [[TMP158:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP159:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP31]]
+; CHECK-NEXT:    [[TMP160:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP161:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP33]]
+; CHECK-NEXT:    [[TMP162:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP34]]
+; CHECK-NEXT:    [[TMP163:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP164:%.*]] = load float, ptr [[TMP132]], align 4
+; CHECK-NEXT:    [[TMP165:%.*]] = load float, ptr [[TMP133]], align 4
+; CHECK-NEXT:    [[TMP166:%.*]] = load float, ptr [[TMP134]], align 4
+; CHECK-NEXT:    [[TMP167:%.*]] = load float, ptr [[TMP135]], align 4
+; CHECK-NEXT:    [[TMP168:%.*]] = load float, ptr [[TMP136]], align 4
+; CHECK-NEXT:    [[TMP169:%.*]] = load float, ptr [[TMP137]], align 4
+; CHECK-NEXT:    [[TMP170:%.*]] = load float, ptr [[TMP138]], align 4
+; CHECK-NEXT:    [[TMP171:%.*]] = load float, ptr [[TMP139]], align 4
+; CHECK-NEXT:    [[TMP172:%.*]] = insertelement <8 x float> poison, float [[TMP164]], i32 0
+; CHECK-NEXT:    [[TMP173:%.*]] = insertelement <8 x float> [[TMP172]], float [[TMP165]], i32 1
+; CHECK-NEXT:    [[TMP174:%.*]] = insertelement <8 x float> [[TMP173]], float [[TMP166]], i32 2
+; CHECK-NEXT:    [[TMP175:%.*]] = insertelement <8 x float> [[TMP174]], float [[TMP167]], i32 3
+; CHECK-NEXT:    [[TMP176:%.*]] = insertelement <8 x float> [[TMP175]], float [[TMP168]], i32 4
+; CHECK-NEXT:    [[TMP177:%.*]] = insertelement <8 x float> [[TMP176]], float [[TMP169]], i32 5
+; CHECK-NEXT:    [[TMP178:%.*]] = insertelement <8 x float> [[TMP177]], float [[TMP170]], i32 6
+; CHECK-NEXT:    [[TMP179:%.*]] = insertelement <8 x float> [[TMP178]], float [[TMP171]], i32 7
+; CHECK-NEXT:    [[TMP180:%.*]] = load float, ptr [[TMP140]], align 4
+; CHECK-NEXT:    [[TMP181:%.*]] = load float, ptr [[TMP141]], align 4
+; CHECK-NEXT:    [[TMP182:%.*]] = load float, ptr [[TMP142]], align 4
+; CHECK-NEXT:    [[TMP183:%.*]] = load float, ptr [[TMP143]], align 4
+; CHECK-NEXT:    [[TMP184:%.*]] = load float, ptr [[TMP144]], align 4
+; CHECK-NEXT:    [[TMP185:%.*]] = load float, ptr [[TMP145]], align 4
+; CHECK-NEXT:    [[TMP186:%.*]] = load float, ptr [[TMP146]], align 4
+; CHECK-NEXT:    [[TMP187:%.*]] = load float, ptr [[TMP147]], align 4
+; CHECK-NEXT:    [[TMP188:%.*]] = insertelement <8 x float> poison, float [[TMP180]], i32 0
+; CHECK-NEXT:    [[TMP189:%.*]] = insertelement <8 x float> [[TMP188]], float [[TMP181]], i32 1
+; CHECK-NEXT:    [[TMP190:%.*]] = insertelement <8 x float> [[TMP189]], float [[TMP182]], i32 2
+; CHECK-NEXT:    [[TMP191:%.*]] = insertelement <8 x float> [[TMP190]], float [[TMP183]], i32 3
+; CHECK-NEXT:    [[TMP192:%.*]] = insertelement <8 x float> [[TMP191]], float [[TMP184]], i32 4
+; CHECK-NEXT:    [[TMP193:%.*]] = insertelement <8 x float> [[TMP192]], float [[TMP185]], i32 5
+; CHECK-NEXT:    [[TMP194:%.*]] = insertelement <8 x float> [[TMP193]], float [[TMP186]], i32 6
+; CHECK-NEXT:    [[TMP195:%.*]] = insertelement <8 x float> [[TMP194]], float [[TMP187]], i32 7
+; CHECK-NEXT:    [[TMP196:%.*]] = load float, ptr [[TMP148]], align 4
+; CHECK-NEXT:    [[TMP197:%.*]] = load float, ptr [[TMP149]], align 4
+; CHECK-NEXT:    [[TMP198:%.*]] = load float, ptr [[TMP150]], align 4
+; CHECK-NEXT:    [[TMP199:%.*]] = load float, ptr [[TMP151]], align 4
+; CHECK-NEXT:    [[TMP200:%.*]] = load float, ptr [[TMP152]], align 4
+; CHECK-NEXT:    [[TMP201:%.*]] = load float, ptr [[TMP153]], align 4
+; CHECK-NEXT:    [[TMP202:%.*]] = load float, ptr [[TMP154]], align 4
+; CHECK-NEXT:    [[TMP203:%.*]] = load float, ptr [[TMP155]], align 4
+; CHECK-NEXT:    [[TMP204:%.*]] = insertelement <8 x float> poison, float [[TMP196]], i32 0
+; CHECK-NEXT:    [[TMP205:%.*]] = insertelement <8 x float> [[TMP204]], float [[TMP197]], i32 1
+; CHECK-NEXT:    [[TMP206:%.*]] = insertelement <8 x float> [[TMP205]], float [[TMP198]], i32 2
+; CHECK-NEXT:    [[TMP207:%.*]] = insertelement <8 x float> [[TMP206]], float [[TMP199]], i32 3
+; CHECK-NEXT:    [[TMP208:%.*]] = insertelement <8 x float> [[TMP207]], float [[TMP200]], i32 4
+; CHECK-NEXT:    [[TMP209:%.*]] = insertelement <8 x float> [[TMP208]], float [[TMP201]], i32 5
+; CHECK-NEXT:    [[TMP210:%.*]] = insertelement <8 x float> [[TMP209]], float [[TMP202]], i32 6
+; CHECK-NEXT:    [[TMP211:%.*]] = insertelement <8 x float> [[TMP210]], float [[TMP203]], i32 7
+; CHECK-NEXT:    [[TMP212:%.*]] = load float, ptr [[TMP156]], align 4
+; CHECK-NEXT:    [[TMP213:%.*]] = load float, ptr [[TMP157]], align 4
+; CHECK-NEXT:    [[TMP214:%.*]] = load float, ptr [[TMP158]], align 4
+; CHECK-NEXT:    [[TMP215:%.*]] = load float, ptr [[TMP159]], align 4
+; CHECK-NEXT:    [[TMP216:%.*]] = load float, ptr [[TMP160]], align 4
+; CHECK-NEXT:    [[TMP217:%.*]] = load float, ptr [[TMP161]], align 4
+; CHECK-NEXT:    [[TMP218:%.*]] = load float, ptr [[TMP162]], align 4
+; CHECK-NEXT:    [[TMP219:%.*]] = load float, ptr [[TMP163]], align 4
+; CHECK-NEXT:    [[TMP220:%.*]] = insertelement <8 x float> poison, float [[TMP212]], i32 0
+; CHECK-NEXT:    [[TMP221:%.*]] = insertelement <8 x float> [[TMP220]], float [[TMP213]], i32 1
+; CHECK-NEXT:    [[TMP222:%.*]] = insertelement <8 x float> [[TMP221]], float [[TMP214]], i32 2
+; CHECK-NEXT:    [[TMP223:%.*]] = insertelement <8 x float> [[TMP222]], float [[TMP215]], i32 3
+; CHECK-NEXT:    [[TMP224:%.*]] = insertelement <8 x float> [[TMP223]], float [[TMP216]], i32 4
+; CHECK-NEXT:    [[TMP225:%.*]] = insertelement <8 x float> [[TMP224]], float [[TMP217]], i32 5
+; CHECK-NEXT:    [[TMP226:%.*]] = insertelement <8 x float> [[TMP225]], float [[TMP218]], i32 6
+; CHECK-NEXT:    [[TMP227:%.*]] = insertelement <8 x float> [[TMP226]], float [[TMP219]], i32 7
+; CHECK-NEXT:    [[TMP228:%.*]] = fadd fast <8 x float> [[TMP83]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP229:%.*]] = fadd fast <8 x float> [[TMP99]], [[VEC_PHI2]]
+; CHECK-NEXT:    [[TMP230:%.*]] = fadd fast <8 x float> [[TMP115]], [[VEC_PHI3]]
+; CHECK-NEXT:    [[TMP231:%.*]] = fadd fast <8 x float> [[TMP131]], [[VEC_PHI4]]
+; CHECK-NEXT:    [[TMP232]] = fadd fast <8 x float> [[TMP228]], [[TMP179]]
+; CHECK-NEXT:    [[TMP233]] = fadd fast <8 x float> [[TMP229]], [[TMP195]]
+; CHECK-NEXT:    [[TMP234]] = fadd fast <8 x float> [[TMP230]], [[TMP211]]
+; CHECK-NEXT:    [[TMP235]] = fadd fast <8 x float> [[TMP231]], [[TMP227]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT:    [[TMP236:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP236]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP121]], [[TMP120]]
-; CHECK-NEXT:    [[BIN_RDX5:%.*]] = fadd fast <4 x float> [[TMP122]], [[BIN_RDX]]
-; CHECK-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <4 x float> [[TMP123]], [[BIN_RDX5]]
-; CHECK-NEXT:    [[TMP125:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX6]])
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP233]], [[TMP232]]
+; CHECK-NEXT:    [[BIN_RDX5:%.*]] = fadd fast <8 x float> [[TMP234]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <8 x float> [[TMP235]], [[BIN_RDX5]]
+; CHECK-NEXT:    [[TMP237:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[BIN_RDX6]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], [[LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
@@ -227,50 +339,50 @@ define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP125]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP237]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[N_MOD_VF7:%.*]] = urem i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_VEC8:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF7]]
-; CHECK-NEXT:    [[TMP126:%.*]] = shl i64 [[N_VEC8]], 5
-; CHECK-NEXT:    [[TMP127:%.*]] = insertelement <4 x float> zeroinitializer, float [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT:    [[TMP238:%.*]] = shl i64 [[N_VEC8]], 5
+; CHECK-NEXT:    [[TMP239:%.*]] = insertelement <4 x float> zeroinitializer, float [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI10:%.*]] = phi <4 x float> [ [[TMP127]], %[[VEC_EPILOG_PH]] ], [ [[TMP157:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP128:%.*]] = shl i64 [[INDEX9]], 5
-; CHECK-NEXT:    [[TMP129:%.*]] = add i64 [[TMP128]], 32
-; CHECK-NEXT:    [[TMP130:%.*]] = add i64 [[TMP128]], 64
-; CHECK-NEXT:    [[TMP131:%.*]] = add i64 [[TMP128]], 96
-; CHECK-NEXT:    [[TMP132:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP128]]
-; CHECK-NEXT:    [[TMP133:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP129]]
-; CHECK-NEXT:    [[TMP134:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP130]]
-; CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP131]]
-; CHECK-NEXT:    [[TMP136:%.*]] = load float, ptr [[TMP132]], align 4
-; CHECK-NEXT:    [[TMP137:%.*]] = load float, ptr [[TMP133]], align 4
-; CHECK-NEXT:    [[TMP138:%.*]] = load float, ptr [[TMP134]], align 4
-; CHECK-NEXT:    [[TMP139:%.*]] = load float, ptr [[TMP135]], align 4
-; CHECK-NEXT:    [[TMP140:%.*]] = insertelement <4 x float> poison, float [[TMP136]], i32 0
-; CHECK-NEXT:    [[TMP141:%.*]] = insertelement <4 x float> [[TMP140]], float [[TMP137]], i32 1
-; CHECK-NEXT:    [[TMP142:%.*]] = insertelement <4 x float> [[TMP141]], float [[TMP138]], i32 2
-; CHECK-NEXT:    [[TMP143:%.*]] = insertelement <4 x float> [[TMP142]], float [[TMP139]], i32 3
-; CHECK-NEXT:    [[TMP144:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP128]]
-; CHECK-NEXT:    [[TMP145:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP129]]
-; CHECK-NEXT:    [[TMP146:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP130]]
-; CHECK-NEXT:    [[TMP147:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP131]]
-; CHECK-NEXT:    [[TMP148:%.*]] = load float, ptr [[TMP144]], align 4
-; CHECK-NEXT:    [[TMP149:%.*]] = load float, ptr [[TMP145]], align 4
-; CHECK-NEXT:    [[TMP150:%.*]] = load float, ptr [[TMP146]], align 4
-; CHECK-NEXT:    [[TMP151:%.*]] = load float, ptr [[TMP147]], align 4
-; CHECK-NEXT:    [[TMP152:%.*]] = insertelement <4 x float> poison, float [[TMP148]], i32 0
-; CHECK-NEXT:    [[TMP153:%.*]] = insertelement <4 x float> [[TMP152]], float [[TMP149]], i32 1
-; CHECK-NEXT:    [[TMP154:%.*]] = insertelement <4 x float> [[TMP153]], float [[TMP150]], i32 2
-; CHECK-NEXT:    [[TMP155:%.*]] = insertelement <4 x float> [[TMP154]], float [[TMP151]], i32 3
-; CHECK-NEXT:    [[TMP156:%.*]] = fadd fast <4 x float> [[TMP143]], [[VEC_PHI10]]
-; CHECK-NEXT:    [[TMP157]] = fadd fast <4 x float> [[TMP156]], [[TMP155]]
+; CHECK-NEXT:    [[VEC_PHI10:%.*]] = phi <4 x float> [ [[TMP239]], %[[VEC_EPILOG_PH]] ], [ [[TMP269:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP240:%.*]] = shl i64 [[INDEX9]], 5
+; CHECK-NEXT:    [[TMP241:%.*]] = add i64 [[TMP240]], 32
+; CHECK-NEXT:    [[TMP242:%.*]] = add i64 [[TMP240]], 64
+; CHECK-NEXT:    [[TMP243:%.*]] = add i64 [[TMP240]], 96
+; CHECK-NEXT:    [[TMP244:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP240]]
+; CHECK-NEXT:    [[TMP245:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP241]]
+; CHECK-NEXT:    [[TMP246:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP242]]
+; CHECK-NEXT:    [[TMP247:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP243]]
+; CHECK-NEXT:    [[TMP248:%.*]] = load float, ptr [[TMP244]], align 4
+; CHECK-NEXT:    [[TMP249:%.*]] = load float, ptr [[TMP245]], align 4
+; CHECK-NEXT:    [[TMP250:%.*]] = load float, ptr [[TMP246]], align 4
+; CHECK-NEXT:    [[TMP251:%.*]] = load float, ptr [[TMP247]], align 4
+; CHECK-NEXT:    [[TMP252:%.*]] = insertelement <4 x float> poison, float [[TMP248]], i32 0
+; CHECK-NEXT:    [[TMP253:%.*]] = insertelement <4 x float> [[TMP252]], float [[TMP249]], i32 1
+; CHECK-NEXT:    [[TMP254:%.*]] = insertelement <4 x float> [[TMP253]], float [[TMP250]], i32 2
+; CHECK-NEXT:    [[TMP255:%.*]] = insertelement <4 x float> [[TMP254]], float [[TMP251]], i32 3
+; CHECK-NEXT:    [[TMP256:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP240]]
+; CHECK-NEXT:    [[TMP257:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP241]]
+; CHECK-NEXT:    [[TMP258:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP242]]
+; CHECK-NEXT:    [[TMP259:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP243]]
+; CHECK-NEXT:    [[TMP260:%.*]] = load float, ptr [[TMP256]], align 4
+; CHECK-NEXT:    [[TMP261:%.*]] = load float, ptr [[TMP257]], align 4
+; CHECK-NEXT:    [[TMP262:%.*]] = load float, ptr [[TMP258]], align 4
+; CHECK-NEXT:    [[TMP263:%.*]] = load float, ptr [[TMP259]], align 4
+; CHECK-NEXT:    [[TMP264:%.*]] = insertelement <4 x float> poison, float [[TMP260]], i32 0
+; CHECK-NEXT:    [[TMP265:%.*]] = insertelement <4 x float> [[TMP264]], float [[TMP261]], i32 1
+; CHECK-NEXT:    [[TMP266:%.*]] = insertelement <4 x float> [[TMP265]], float [[TMP262]], i32 2
+; CHECK-NEXT:    [[TMP267:%.*]] = insertelement <4 x float> [[TMP266]], float [[TMP263]], i32 3
+; CHECK-NEXT:    [[TMP268:%.*]] = fadd fast <4 x float> [[TMP255]], [[VEC_PHI10]]
+; CHECK-NEXT:    [[TMP269]] = fadd fast <4 x float> [[TMP268]], [[TMP267]]
 ; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], 4
-; CHECK-NEXT:    [[TMP158:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC8]]
-; CHECK-NEXT:    br i1 [[TMP158]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP270:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC8]]
+; CHECK-NEXT:    br i1 [[TMP270]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP159:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP157]])
+; CHECK-NEXT:    [[TMP271:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP269]])
 ; CHECK-NEXT:    [[CMP_N12:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC8]]
 ; CHECK-NEXT:    br i1 [[CMP_N12]], [[LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
@@ -475,7 +587,7 @@ loop:
   %any.of.next = select i1 %cmp13.not.not, i1 %any.of, i1 true
   %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 40
   %cmp.not = icmp eq ptr %ptr.iv, %end
-  br i1 %cmp.not, label %exit, label %loop
+  br i1 %cmp.not, label %exit, label %loop, !llvm.loop !0
 
 exit:
   ret i1 %any.of.next
@@ -795,7 +907,6 @@ define i32 @g(i64 %n) {
 ; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
-; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF24:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
@@ -819,7 +930,7 @@ define i32 @g(i64 %n) {
 ; CHECK-NEXT:    [[INDEX_NEXT16]] = add nuw i32 [[INDEX13]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT17]] = add <4 x i32> [[VEC_IND14]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT16]], [[N_VEC8]]
-; CHECK-NEXT:    br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP25]])
 ; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC8]]
@@ -943,7 +1054,7 @@ define void @known_deref_load_tail_folding() #4 {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
@@ -973,3 +1084,8 @@ attributes #1 = { "target-features"="+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx
 attributes #2 = { "target-cpu"="znver3" }
 attributes #3 = { "target-cpu"="skylake-avx512" }
 attributes #4 = { "target-cpu"="haswell" }
+
+!0 = distinct !{!0, !1, !2, !3}
+!1 = !{!"llvm.loop.vectorize.width", i32 2}
+!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
index 80a2e73be4197..a770eee1ef1ae 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S %s | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll b/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
index c23ea3373cb6a..9823d18da2905 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
@@ -164,7 +164,7 @@ loop:
   %dst.gep = getelementptr i32, ptr %A, i64 %iv.next
   store i32 %store.val, ptr %dst.gep, align 4
   %exit.cond = icmp eq i64 %iv, 0
-  br i1 %exit.cond, label %exit, label %loop
+  br i1 %exit.cond, label %exit, label %loop, !llvm.loop !0
 
 exit:
   %liveout = phi i32 [ %load, %loop ]
@@ -172,3 +172,8 @@ exit:
 }
 
 attributes #0 = { "target-cpu"="skylake-avx512" }
+
+!0 = distinct !{!0, !1, !2, !3}
+!1 = !{!"llvm.loop.vectorize.width", i32 2}
+!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index c9e099f4b2f21..f1b49668f2b10 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -189,21 +189,14 @@ define void @test_scalar_cost_single_store_loop_varying_cond(ptr %dst, ptr noali
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 16
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i32> [[STRIDED_VEC]], splat (i32 123)
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[STRIDED_VEC2]], splat (i32 123)
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <4 x i1> [[TMP5]])
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr align 4 [[TMP7]], <4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <8 x i32> [[STRIDED_VEC]], splat (i32 123)
+; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[TMP3]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH:.*]]
 ; CHECK:       [[SCALAR_PH]]:

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 1b112b6ba9a0f..7d04d2759540f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -1073,46 +1073,27 @@ for.end:
 define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr nocapture readonly %trigger) {
 ; AVX1-LABEL: define void @foo6(
 ; AVX1-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) #[[ATTR0]] {
-; AVX1-NEXT:  [[ENTRY:.*:]]
-; AVX1-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
-; AVX1:       [[VECTOR_MEMCHECK]]:
-; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 32768
-; AVX1-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TRIGGER]], i64 16384
-; AVX1-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[IN]], i64 32768
-; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
-; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TRIGGER]], [[SCEVGEP]]
-; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; AVX1-NEXT:    [[BOUND03:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP2]]
-; AVX1-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]]
-; AVX1-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
-; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
-; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; AVX1:       [[VECTOR_PH]]:
-; AVX1-NEXT:    br label %[[VECTOR_BODY:.*]]
-; AVX1:       [[VECTOR_BODY]]:
-; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX1-NEXT:    [[TMP0:%.*]] = sub i64 4095, [[INDEX]]
-; AVX1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP0]]
-; AVX1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -3
-; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META18:![0-9]+]]
-; AVX1-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; AVX1-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[REVERSE]], zeroinitializer
-; AVX1-NEXT:    [[TMP4:%.*]] = getelementptr double, ptr [[IN]], i64 [[TMP0]]
-; AVX1-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[TMP4]], i64 -3
-; AVX1-NEXT:    [[REVERSE6:%.*]] = shufflevector <4 x i1> [[TMP3]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP5]], <4 x i1> [[REVERSE6]], <4 x double> poison), !alias.scope [[META21:![0-9]+]]
-; AVX1-NEXT:    [[REVERSE7:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; AVX1-NEXT:    [[TMP6:%.*]] = fadd <4 x double> [[REVERSE7]], splat (double 5.000000e-01)
-; AVX1-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[OUT]], i64 [[TMP0]]
-; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[TMP7]], i64 -3
-; AVX1-NEXT:    [[REVERSE8:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE8]], ptr align 8 [[TMP8]], <4 x i1> [[REVERSE6]]), !alias.scope [[META23:![0-9]+]], !noalias [[META25:![0-9]+]]
-; AVX1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; AVX1-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; AVX1-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
-; AVX1:       [[MIDDLE_BLOCK]]:
-; AVX1-NEXT:    br [[FOR_END:label %.*]]
-; AVX1:       [[SCALAR_PH]]:
+; AVX1-NEXT:  [[ENTRY:.*]]:
+; AVX1-NEXT:    br label %[[FOR_BODY:.*]]
+; AVX1:       [[FOR_BODY]]:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 4095, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; AVX1-NEXT:    br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; AVX1:       [[IF_THEN]]:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[IN]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
+; AVX1-NEXT:    [[ADD:%.*]] = fadd double [[TMP1]], 5.000000e-01
+; AVX1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store double [[ADD]], ptr [[ARRAYIDX5]], align 8
+; AVX1-NEXT:    br label %[[FOR_INC]]
+; AVX1:       [[FOR_INC]]:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; AVX1-NEXT:    [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; AVX1-NEXT:    br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
+; AVX1:       [[FOR_END]]:
+; AVX1-NEXT:    ret void
 ;
 ; AVX2-LABEL: define void @foo6(
 ; AVX2-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) #[[ATTR0]] {
@@ -1368,13 +1349,13 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <4 x i1> [[TMP23]])
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; AVX1-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; AVX1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; AVX1:       [[MIDDLE_BLOCK]]:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX1-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; AVX1:       [[VEC_EPILOG_ITER_CHECK]]:
 ; AVX1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
-; AVX1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29:![0-9]+]]
+; AVX1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19:![0-9]+]]
 ; AVX1:       [[VEC_EPILOG_PH]]:
 ; AVX1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; AVX1-NEXT:    [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
@@ -1394,7 +1375,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <4 x i1> [[TMP34]])
 ; AVX1-NEXT:    [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
 ; AVX1-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; AVX1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; AVX1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
 ; AVX1-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX1-NEXT:    br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
@@ -1689,13 +1670,13 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <4 x i1> [[TMP23]])
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; AVX1-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; AVX1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; AVX1:       [[MIDDLE_BLOCK]]:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX1-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; AVX1:       [[VEC_EPILOG_ITER_CHECK]]:
 ; AVX1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
-; AVX1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29]]
+; AVX1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19]]
 ; AVX1:       [[VEC_EPILOG_PH]]:
 ; AVX1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; AVX1-NEXT:    [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
@@ -1715,7 +1696,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <4 x i1> [[TMP34]])
 ; AVX1-NEXT:    [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
 ; AVX1-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]]
-; AVX1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; AVX1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; AVX1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
 ; AVX1-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
 ; AVX1-NEXT:    br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index 309e9a90a3c99..4802df6caee6d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -789,44 +789,10 @@ exit:
 define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; COST-LABEL: define void @switch4_default_common_dest_with_case(
 ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
-; COST-NEXT:  [[ENTRY:.*]]:
-; COST-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64
-; COST-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
-; COST-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
-; COST-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
-; COST-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
-; COST-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; COST-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; COST-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST:       [[VECTOR_PH]]:
-; COST-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
-; COST-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; COST-NEXT:    [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
-; COST-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; COST-NEXT:    br label %[[VECTOR_BODY:.*]]
-; COST:       [[VECTOR_BODY]]:
-; COST-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 3
-; COST-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; COST-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
-; COST-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
-; COST-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
-; COST-NEXT:    [[TMP9:%.*]] = or <4 x i1> [[TMP7]], [[TMP8]]
-; COST-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP8]])
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP10]])
-; COST-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; COST:       [[MIDDLE_BLOCK]]:
-; COST-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; COST-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COST:       [[SCALAR_PH]]:
-; COST-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT:  [[SCALAR_PH:.*]]:
 ; COST-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; COST:       [[LOOP_HEADER]]:
-; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; COST-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
 ; COST-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
 ; COST-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
@@ -845,7 +811,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; COST:       [[LOOP_LATCH]]:
 ; COST-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
 ; COST-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; COST-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
 ; COST:       [[EXIT]]:
 ; COST-NEXT:    ret void
 ;
@@ -956,50 +922,10 @@ exit:
 define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
 ; COST-LABEL: define void @switch_under_br_default_common_dest_with_case(
 ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
-; COST-NEXT:  [[ENTRY:.*]]:
-; COST-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64
-; COST-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
-; COST-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
-; COST-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
-; COST-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
-; COST-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; COST-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; COST-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST:       [[VECTOR_PH]]:
-; COST-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
-; COST-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; COST-NEXT:    [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
-; COST-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
-; COST-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
-; COST-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COST-NEXT:    br label %[[VECTOR_BODY:.*]]
-; COST:       [[VECTOR_BODY]]:
-; COST-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 3
-; COST-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; COST-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
-; COST-NEXT:    [[TMP7:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; COST-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
-; COST-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
-; COST-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
-; COST-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
-; COST-NEXT:    [[TMP12:%.*]] = or <4 x i1> [[TMP10]], [[TMP11]]
-; COST-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP12]], splat (i1 true)
-; COST-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP13]], <4 x i1> zeroinitializer
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]])
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP10]])
-; COST-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
-; COST-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; COST:       [[MIDDLE_BLOCK]]:
-; COST-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; COST-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COST:       [[SCALAR_PH]]:
-; COST-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; COST-NEXT:  [[SCALAR_PH:.*]]:
 ; COST-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; COST:       [[LOOP_HEADER]]:
-; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; COST-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
 ; COST-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], [[X]]
 ; COST-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
@@ -1021,7 +947,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end,
 ; COST:       [[LOOP_LATCH]]:
 ; COST-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
 ; COST-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; COST-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
 ; COST:       [[EXIT]]:
 ; COST-NEXT:    ret void
 ;
@@ -1149,10 +1075,10 @@ exit:
 define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end, i64 %x) {
 ; COST-LABEL: define void @br_under_switch_default_common_dest_with_case(
 ; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
-; COST-NEXT:  [[SCALAR_PH:.*]]:
-; COST-NEXT:    br label %[[LOOP_HEADER1:.*]]
-; COST:       [[LOOP_HEADER1]]:
-; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT:  [[ENTRY:.*]]:
+; COST-NEXT:    br label %[[LOOP_HEADER:.*]]
+; COST:       [[LOOP_HEADER]]:
+; COST-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; COST-NEXT:    [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
 ; COST-NEXT:    switch i64 [[L]], label %[[DEFAULT:.*]] [
 ; COST-NEXT:      i64 -12, label %[[IF_THEN_1:.*]]
@@ -1174,7 +1100,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
 ; COST:       [[LOOP_LATCH]]:
 ; COST-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
 ; COST-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]]
+; COST-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
 ; COST:       [[EXIT]]:
 ; COST-NEXT:    ret void
 ;
@@ -1455,4 +1381,3 @@ loop.latch:
 exit:
   ret void
 }
-

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 394e972d79a86..04ca0ec9d4c57 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -571,28 +571,60 @@ define void @test(ptr %A, ptr noalias %B) #0 {
 ; MAX-BW-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26
 ; MAX-BW-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28
 ; MAX-BW-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30
+; MAX-BW-NEXT:    [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 32
+; MAX-BW-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 34
+; MAX-BW-NEXT:    [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 36
+; MAX-BW-NEXT:    [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], 38
+; MAX-BW-NEXT:    [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 40
+; MAX-BW-NEXT:    [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 42
+; MAX-BW-NEXT:    [[TMP22:%.*]] = add i64 [[OFFSET_IDX]], 44
+; MAX-BW-NEXT:    [[TMP23:%.*]] = add i64 [[OFFSET_IDX]], 46
+; MAX-BW-NEXT:    [[TMP24:%.*]] = add i64 [[OFFSET_IDX]], 48
+; MAX-BW-NEXT:    [[TMP25:%.*]] = add i64 [[OFFSET_IDX]], 50
+; MAX-BW-NEXT:    [[TMP26:%.*]] = add i64 [[OFFSET_IDX]], 52
+; MAX-BW-NEXT:    [[TMP27:%.*]] = add i64 [[OFFSET_IDX]], 54
+; MAX-BW-NEXT:    [[TMP28:%.*]] = add i64 [[OFFSET_IDX]], 56
+; MAX-BW-NEXT:    [[TMP29:%.*]] = add i64 [[OFFSET_IDX]], 58
+; MAX-BW-NEXT:    [[TMP30:%.*]] = add i64 [[OFFSET_IDX]], 60
+; MAX-BW-NEXT:    [[TMP31:%.*]] = add i64 [[OFFSET_IDX]], 62
 ; MAX-BW-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[OFFSET_IDX]]
-; MAX-BW-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i32>, ptr [[TMP32]], align 4
-; MAX-BW-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; MAX-BW-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; MAX-BW-NEXT:    [[TMP34:%.*]] = add <16 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
-; MAX-BW-NEXT:    [[TMP35:%.*]] = trunc <16 x i32> [[TMP34]] to <16 x i8>
-; MAX-BW-NEXT:    [[TMP52:%.*]] = extractelement <16 x i8> [[TMP35]], i64 0
-; MAX-BW-NEXT:    [[TMP53:%.*]] = extractelement <16 x i8> [[TMP35]], i64 1
-; MAX-BW-NEXT:    [[TMP54:%.*]] = extractelement <16 x i8> [[TMP35]], i64 2
-; MAX-BW-NEXT:    [[TMP55:%.*]] = extractelement <16 x i8> [[TMP35]], i64 3
-; MAX-BW-NEXT:    [[TMP56:%.*]] = extractelement <16 x i8> [[TMP35]], i64 4
-; MAX-BW-NEXT:    [[TMP57:%.*]] = extractelement <16 x i8> [[TMP35]], i64 5
-; MAX-BW-NEXT:    [[TMP58:%.*]] = extractelement <16 x i8> [[TMP35]], i64 6
-; MAX-BW-NEXT:    [[TMP59:%.*]] = extractelement <16 x i8> [[TMP35]], i64 7
-; MAX-BW-NEXT:    [[TMP60:%.*]] = extractelement <16 x i8> [[TMP35]], i64 8
-; MAX-BW-NEXT:    [[TMP61:%.*]] = extractelement <16 x i8> [[TMP35]], i64 9
-; MAX-BW-NEXT:    [[TMP62:%.*]] = extractelement <16 x i8> [[TMP35]], i64 10
-; MAX-BW-NEXT:    [[TMP63:%.*]] = extractelement <16 x i8> [[TMP35]], i64 11
-; MAX-BW-NEXT:    [[TMP64:%.*]] = extractelement <16 x i8> [[TMP35]], i64 12
-; MAX-BW-NEXT:    [[TMP65:%.*]] = extractelement <16 x i8> [[TMP35]], i64 13
-; MAX-BW-NEXT:    [[TMP66:%.*]] = extractelement <16 x i8> [[TMP35]], i64 14
-; MAX-BW-NEXT:    [[TMP67:%.*]] = extractelement <16 x i8> [[TMP35]], i64 15
+; MAX-BW-NEXT:    [[WIDE_VEC:%.*]] = load <64 x i32>, ptr [[TMP32]], align 4
+; MAX-BW-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
+; MAX-BW-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
+; MAX-BW-NEXT:    [[TMP33:%.*]] = add <32 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; MAX-BW-NEXT:    [[TMP34:%.*]] = trunc <32 x i32> [[TMP33]] to <32 x i8>
+; MAX-BW-NEXT:    [[TMP52:%.*]] = extractelement <32 x i8> [[TMP34]], i64 0
+; MAX-BW-NEXT:    [[TMP53:%.*]] = extractelement <32 x i8> [[TMP34]], i64 1
+; MAX-BW-NEXT:    [[TMP54:%.*]] = extractelement <32 x i8> [[TMP34]], i64 2
+; MAX-BW-NEXT:    [[TMP55:%.*]] = extractelement <32 x i8> [[TMP34]], i64 3
+; MAX-BW-NEXT:    [[TMP56:%.*]] = extractelement <32 x i8> [[TMP34]], i64 4
+; MAX-BW-NEXT:    [[TMP57:%.*]] = extractelement <32 x i8> [[TMP34]], i64 5
+; MAX-BW-NEXT:    [[TMP58:%.*]] = extractelement <32 x i8> [[TMP34]], i64 6
+; MAX-BW-NEXT:    [[TMP59:%.*]] = extractelement <32 x i8> [[TMP34]], i64 7
+; MAX-BW-NEXT:    [[TMP60:%.*]] = extractelement <32 x i8> [[TMP34]], i64 8
+; MAX-BW-NEXT:    [[TMP61:%.*]] = extractelement <32 x i8> [[TMP34]], i64 9
+; MAX-BW-NEXT:    [[TMP62:%.*]] = extractelement <32 x i8> [[TMP34]], i64 10
+; MAX-BW-NEXT:    [[TMP63:%.*]] = extractelement <32 x i8> [[TMP34]], i64 11
+; MAX-BW-NEXT:    [[TMP64:%.*]] = extractelement <32 x i8> [[TMP34]], i64 12
+; MAX-BW-NEXT:    [[TMP65:%.*]] = extractelement <32 x i8> [[TMP34]], i64 13
+; MAX-BW-NEXT:    [[TMP66:%.*]] = extractelement <32 x i8> [[TMP34]], i64 14
+; MAX-BW-NEXT:    [[TMP67:%.*]] = extractelement <32 x i8> [[TMP34]], i64 15
+; MAX-BW-NEXT:    [[TMP99:%.*]] = extractelement <32 x i8> [[TMP34]], i64 16
+; MAX-BW-NEXT:    [[TMP100:%.*]] = extractelement <32 x i8> [[TMP34]], i64 17
+; MAX-BW-NEXT:    [[TMP101:%.*]] = extractelement <32 x i8> [[TMP34]], i64 18
+; MAX-BW-NEXT:    [[TMP102:%.*]] = extractelement <32 x i8> [[TMP34]], i64 19
+; MAX-BW-NEXT:    [[TMP103:%.*]] = extractelement <32 x i8> [[TMP34]], i64 20
+; MAX-BW-NEXT:    [[TMP104:%.*]] = extractelement <32 x i8> [[TMP34]], i64 21
+; MAX-BW-NEXT:    [[TMP105:%.*]] = extractelement <32 x i8> [[TMP34]], i64 22
+; MAX-BW-NEXT:    [[TMP106:%.*]] = extractelement <32 x i8> [[TMP34]], i64 23
+; MAX-BW-NEXT:    [[TMP107:%.*]] = extractelement <32 x i8> [[TMP34]], i64 24
+; MAX-BW-NEXT:    [[TMP108:%.*]] = extractelement <32 x i8> [[TMP34]], i64 25
+; MAX-BW-NEXT:    [[TMP109:%.*]] = extractelement <32 x i8> [[TMP34]], i64 26
+; MAX-BW-NEXT:    [[TMP110:%.*]] = extractelement <32 x i8> [[TMP34]], i64 27
+; MAX-BW-NEXT:    [[TMP111:%.*]] = extractelement <32 x i8> [[TMP34]], i64 28
+; MAX-BW-NEXT:    [[TMP112:%.*]] = extractelement <32 x i8> [[TMP34]], i64 29
+; MAX-BW-NEXT:    [[TMP113:%.*]] = extractelement <32 x i8> [[TMP34]], i64 30
+; MAX-BW-NEXT:    [[TMP114:%.*]] = extractelement <32 x i8> [[TMP34]], i64 31
 ; MAX-BW-NEXT:    [[TMP69:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[OFFSET_IDX]]
 ; MAX-BW-NEXT:    [[TMP70:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP1]]
 ; MAX-BW-NEXT:    [[TMP71:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP2]]
@@ -609,6 +641,22 @@ define void @test(ptr %A, ptr noalias %B) #0 {
 ; MAX-BW-NEXT:    [[TMP82:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP13]]
 ; MAX-BW-NEXT:    [[TMP83:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP14]]
 ; MAX-BW-NEXT:    [[TMP51:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP15]]
+; MAX-BW-NEXT:    [[TMP115:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP16]]
+; MAX-BW-NEXT:    [[TMP84:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP17]]
+; MAX-BW-NEXT:    [[TMP85:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP18]]
+; MAX-BW-NEXT:    [[TMP86:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP19]]
+; MAX-BW-NEXT:    [[TMP87:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP20]]
+; MAX-BW-NEXT:    [[TMP88:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP21]]
+; MAX-BW-NEXT:    [[TMP89:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP22]]
+; MAX-BW-NEXT:    [[TMP90:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP23]]
+; MAX-BW-NEXT:    [[TMP91:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP24]]
+; MAX-BW-NEXT:    [[TMP92:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP25]]
+; MAX-BW-NEXT:    [[TMP93:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP26]]
+; MAX-BW-NEXT:    [[TMP94:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP27]]
+; MAX-BW-NEXT:    [[TMP95:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP28]]
+; MAX-BW-NEXT:    [[TMP96:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP29]]
+; MAX-BW-NEXT:    [[TMP97:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP30]]
+; MAX-BW-NEXT:    [[TMP98:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[TMP31]]
 ; MAX-BW-NEXT:    store i8 [[TMP52]], ptr [[TMP69]], align 1
 ; MAX-BW-NEXT:    store i8 [[TMP53]], ptr [[TMP70]], align 1
 ; MAX-BW-NEXT:    store i8 [[TMP54]], ptr [[TMP71]], align 1
@@ -625,7 +673,23 @@ define void @test(ptr %A, ptr noalias %B) #0 {
 ; MAX-BW-NEXT:    store i8 [[TMP65]], ptr [[TMP82]], align 1
 ; MAX-BW-NEXT:    store i8 [[TMP66]], ptr [[TMP83]], align 1
 ; MAX-BW-NEXT:    store i8 [[TMP67]], ptr [[TMP51]], align 1
-; MAX-BW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; MAX-BW-NEXT:    store i8 [[TMP99]], ptr [[TMP115]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP100]], ptr [[TMP84]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP101]], ptr [[TMP85]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP102]], ptr [[TMP86]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP103]], ptr [[TMP87]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP104]], ptr [[TMP88]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP105]], ptr [[TMP89]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP106]], ptr [[TMP90]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP107]], ptr [[TMP91]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP108]], ptr [[TMP92]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP109]], ptr [[TMP93]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP110]], ptr [[TMP94]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP111]], ptr [[TMP95]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP112]], ptr [[TMP96]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP113]], ptr [[TMP97]], align 1
+; MAX-BW-NEXT:    store i8 [[TMP114]], ptr [[TMP98]], align 1
+; MAX-BW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; MAX-BW-NEXT:    [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
 ; MAX-BW-NEXT:    br i1 [[TMP68]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; MAX-BW:       [[MIDDLE_BLOCK]]:

diff  --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
index af512d4949ce4..975a290de5f6a 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
@@ -1085,42 +1085,13 @@ define void @saxpy_5(i64 %n, float %a, ptr readonly %x, ptr noalias %y) {
 ; CHECK-SAME: i64 [[N:%.*]], float [[A:%.*]], ptr readonly captures(none) [[X:%.*]], ptr noalias captures(none) [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
-; CHECK:       [[LOOP_PREHEADER]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[N]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 [[TMP1]], 5
-; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 6
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[LOOP_PREHEADER11:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775806
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 5
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[A]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <10 x i32> zeroinitializer
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <10 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[Y]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[WIDE_VEC5:%.*]] = load <10 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <10 x float> [[WIDE_VEC]], [[TMP5]]
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = fadd fast <10 x float> [[WIDE_VEC5]], [[TMP8]]
-; CHECK-NEXT:    store <10 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT]], label %[[LOOP_PREHEADER11]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[LOOP_PREHEADER11:.*]], label %[[EXIT:.*]]
 ; CHECK:       [[LOOP_PREHEADER11]]:
-; CHECK-NEXT:    [[I1_PH:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> poison, float [[A]], i64 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ [[I1_PH]], %[[LOOP_PREHEADER11]] ]
+; CHECK-NEXT:    [[I1:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER11]] ]
 ; CHECK-NEXT:    [[XGEP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i64 [[I1]]
 ; CHECK-NEXT:    [[YGEP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[Y]], i64 [[I1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x float>, ptr [[XGEP1]], align 4
@@ -1138,7 +1109,7 @@ define void @saxpy_5(i64 %n, float %a, ptr readonly %x, ptr noalias %y) {
 ; CHECK-NEXT:    store float [[AXPY5]], ptr [[YGEP5]], align 4
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I1]], 5
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[N]], [[I_NEXT]]
-; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1204,6 +1175,4 @@ exit:
 ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
 ;.