[llvm] [VPlan] Don't quadratically cost address computation for scalarized mem ops (PR #209830)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 09:49:40 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/209830
>From df452634cf7890fae676990e91509af38fe8a220 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 16 Jul 2026 00:47:08 +0800
Subject: [PATCH] [VPlan] Don't quadratically cost address computation for
scalarized mem ops
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +-
.../AArch64/aggressive-interleaving.ll | 213 +++++++++++---
.../replicating-load-store-costs-apple.ll | 184 ++++++++++--
.../AArch64/replicating-load-store-costs.ll | 184 ++++++++++--
...row-interleave-to-widen-memory-scalable.ll | 47 ++-
.../ARM/replicating-load-store-costs.ll | 86 +++++-
.../X86/CostModel/gather-i16-with-i8-index.ll | 18 +-
.../X86/CostModel/gather-i32-with-i8-index.ll | 26 +-
.../X86/CostModel/gather-i64-with-i8-index.ll | 26 +-
.../X86/CostModel/gather-i8-with-i8-index.ll | 26 +-
.../CostModel/scatter-i16-with-i8-index.ll | 26 +-
.../CostModel/scatter-i32-with-i8-index.ll | 26 +-
.../CostModel/scatter-i64-with-i8-index.ll | 26 +-
.../X86/CostModel/scatter-i8-with-i8-index.ll | 26 +-
.../LoopVectorize/X86/masked_load_store.ll | 273 ++++++++++++++----
.../LoopVectorize/X86/reduction-crash.ll | 100 ++++++-
...licating-load-store-costs-max-bandwidth.ll | 268 ++++++++++++++++-
.../X86/replicating-load-store-costs.ll | 2 +-
19 files changed, 1295 insertions(+), 276 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..89b7726858b12 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4237,9 +4237,6 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
unsigned AS = getLoadStoreAddressSpace(I);
Value *Ptr = getLoadStorePointerOperand(I);
- Type *PtrTy = toVectorTy(Ptr->getType(), VF);
- // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
- // that it is being called from this specific place.
// Figure out whether the access is strided and get the stride value
// if it's known in compile time
@@ -4247,8 +4244,8 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
// Get the cost of the scalar memory instruction and address computation.
InstructionCost Cost =
- VF.getFixedValue() *
- TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV, Config.CostKind);
+ VF.getFixedValue() * TTI.getAddressComputationCost(
+ Ptr->getType(), SE, PtrSCEV, Config.CostKind);
// Don't pass *I here, since it is scalar but will actually be part of a
// vectorized loop where the user of it is a vectorized instruction.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3fbeb7e772a2e..fd342e10cf689 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3900,17 +3900,16 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
UI->getOpcode(), ValTy, Alignment, AS, Ctx.CostKind, OpInfo,
UsedByLoadStoreAddress ? UI : nullptr);
- Type *PtrTy = isSingleScalar() ? ScalarPtrTy : toVectorTy(ScalarPtrTy, VF);
InstructionCost ScalarCost =
ScalarMemOpCost +
Ctx.TTI.getAddressComputationCost(
- PtrTy, UsedByLoadStoreAddress ? nullptr : Ctx.PSE.getSE(), PtrSCEV,
- Ctx.CostKind);
+ ScalarPtrTy, UsedByLoadStoreAddress ? nullptr : Ctx.PSE.getSE(),
+ PtrSCEV, Ctx.CostKind);
if (isSingleScalar())
return ScalarCost;
SmallVector<const VPValue *> OpsToScalarize;
- Type *ResultTy = Type::getVoidTy(PtrTy->getContext());
+ Type *ResultTy = Type::getVoidTy(ScalarPtrTy->getContext());
// Set ResultTy and OpsToScalarize, if scalarization is needed. Currently we
// don't assign scalarization overhead in general, if the target prefers
// vectorized addressing or the loaded value is used as part of an address
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
index a9a91519e7d32..d273eff513ab6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
@@ -17,21 +17,24 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320: [[OUTER]]:
; A320-NEXT: [[TPM26:%.*]] = add i64 0, 1
; A320-NEXT: [[TPM10:%.*]] = alloca i32, align 8
+; A320-NEXT: [[TPM102:%.*]] = ptrtoint ptr [[TPM10]] to i64
; A320-NEXT: [[TPM27:%.*]] = getelementptr inbounds i32, ptr [[TPM10]], i64 [[TPM26]]
; A320-NEXT: [[TPM28:%.*]] = getelementptr inbounds ptr, ptr [[TPM15]], i64 0
; A320-NEXT: [[TPM29:%.*]] = load ptr, ptr [[TPM28]], align 8
+; A320-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[TPM29]] to i64
; A320-NEXT: [[TPM17:%.*]] = alloca double, align 8
; A320-NEXT: [[TPM32:%.*]] = getelementptr inbounds double, ptr [[TPM17]], i64 [[TPM26]]
-; A320-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[TPM29]] to i64
-; A320-NEXT: [[TPM102:%.*]] = ptrtoint ptr [[TPM10]] to i64
-; A320-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[TPM102]]
; A320-NEXT: [[TMP6:%.*]] = add i64 [[TMP1]], -8
-; A320-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP6]], 2
+; A320-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[TPM102]]
+; A320-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP7]], 2
; A320-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; A320-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP3]], 16
+; A320-NEXT: br i1 [[MIN_ITERS_CHECK3]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; A320: [[VECTOR_PH1]]:
+; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 16
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; A320-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
; A320-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP4]]
@@ -39,47 +42,173 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[IND_END3:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP5]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
-; A320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; A320-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
-; A320-NEXT: [[VEC_PHI5:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP88:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI4:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP89:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI5:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP90:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI6:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP91:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
-; A320-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 4
; A320-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[OFFSET_IDX]]
-; A320-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP7]]
; A320-NEXT: [[OFFSET_IDX7:%.*]] = shl i64 [[INDEX]], 3
-; A320-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX7]], 8
-; A320-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[OFFSET_IDX7]]
-; A320-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP9]]
+; A320-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[OFFSET_IDX7]]
+; A320-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 4
+; A320-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 8
+; A320-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 12
+; A320-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[NEXT_GEP7]], align 8
+; A320-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP21]], align 8
+; A320-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP23]], align 8
+; A320-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP12]], align 8
+; A320-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 4
+; A320-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 8
+; A320-NEXT: [[TMP53:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 12
+; A320-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
+; A320-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; A320-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP52]], align 4
+; A320-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP53]], align 4
+; A320-NEXT: [[TMP102:%.*]] = zext <4 x i32> [[WIDE_LOAD11]] to <4 x i64>
+; A320-NEXT: [[TMP103:%.*]] = zext <4 x i32> [[WIDE_LOAD12]] to <4 x i64>
+; A320-NEXT: [[TMP104:%.*]] = zext <4 x i32> [[WIDE_LOAD13]] to <4 x i64>
+; A320-NEXT: [[TMP105:%.*]] = zext <4 x i32> [[WIDE_LOAD14]] to <4 x i64>
+; A320-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP102]], i64 0
+; A320-NEXT: [[NEXT_GEP8:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP20]]
+; A320-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP102]], i64 1
+; A320-NEXT: [[NEXT_GEP9:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP22]]
+; A320-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP102]], i64 2
+; A320-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP24]]
+; A320-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP102]], i64 3
+; A320-NEXT: [[TMP27:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP26]]
+; A320-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[TMP103]], i64 0
+; A320-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP28]]
+; A320-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP103]], i64 1
+; A320-NEXT: [[TMP31:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP30]]
+; A320-NEXT: [[TMP32:%.*]] = extractelement <4 x i64> [[TMP103]], i64 2
+; A320-NEXT: [[TMP33:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP32]]
+; A320-NEXT: [[TMP34:%.*]] = extractelement <4 x i64> [[TMP103]], i64 3
+; A320-NEXT: [[TMP35:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP34]]
+; A320-NEXT: [[TMP36:%.*]] = extractelement <4 x i64> [[TMP104]], i64 0
+; A320-NEXT: [[TMP37:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP36]]
+; A320-NEXT: [[TMP38:%.*]] = extractelement <4 x i64> [[TMP104]], i64 1
+; A320-NEXT: [[TMP39:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP38]]
+; A320-NEXT: [[TMP40:%.*]] = extractelement <4 x i64> [[TMP104]], i64 2
+; A320-NEXT: [[TMP41:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP40]]
+; A320-NEXT: [[TMP42:%.*]] = extractelement <4 x i64> [[TMP104]], i64 3
+; A320-NEXT: [[TMP43:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP42]]
+; A320-NEXT: [[TMP44:%.*]] = extractelement <4 x i64> [[TMP105]], i64 0
+; A320-NEXT: [[TMP45:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP44]]
+; A320-NEXT: [[TMP46:%.*]] = extractelement <4 x i64> [[TMP105]], i64 1
+; A320-NEXT: [[TMP47:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP46]]
+; A320-NEXT: [[TMP48:%.*]] = extractelement <4 x i64> [[TMP105]], i64 2
+; A320-NEXT: [[TMP49:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP48]]
+; A320-NEXT: [[TMP50:%.*]] = extractelement <4 x i64> [[TMP105]], i64 3
+; A320-NEXT: [[TMP51:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP50]]
; A320-NEXT: [[TMP10:%.*]] = load double, ptr [[NEXT_GEP8]], align 8
; A320-NEXT: [[TMP11:%.*]] = load double, ptr [[NEXT_GEP9]], align 8
-; A320-NEXT: [[TMP12:%.*]] = load i32, ptr [[NEXT_GEP]], align 4
-; A320-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP6]], align 4
-; A320-NEXT: [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
-; A320-NEXT: [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
+; A320-NEXT: [[TMP54:%.*]] = load double, ptr [[TMP25]], align 8
+; A320-NEXT: [[TMP55:%.*]] = load double, ptr [[TMP27]], align 8
+; A320-NEXT: [[TMP56:%.*]] = insertelement <4 x double> poison, double [[TMP10]], i32 0
+; A320-NEXT: [[TMP57:%.*]] = insertelement <4 x double> [[TMP56]], double [[TMP11]], i32 1
+; A320-NEXT: [[TMP58:%.*]] = insertelement <4 x double> [[TMP57]], double [[TMP54]], i32 2
+; A320-NEXT: [[TMP59:%.*]] = insertelement <4 x double> [[TMP58]], double [[TMP55]], i32 3
+; A320-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP29]], align 8
+; A320-NEXT: [[TMP61:%.*]] = load double, ptr [[TMP31]], align 8
+; A320-NEXT: [[TMP62:%.*]] = load double, ptr [[TMP33]], align 8
+; A320-NEXT: [[TMP63:%.*]] = load double, ptr [[TMP35]], align 8
+; A320-NEXT: [[TMP64:%.*]] = insertelement <4 x double> poison, double [[TMP60]], i32 0
+; A320-NEXT: [[TMP65:%.*]] = insertelement <4 x double> [[TMP64]], double [[TMP61]], i32 1
+; A320-NEXT: [[TMP66:%.*]] = insertelement <4 x double> [[TMP65]], double [[TMP62]], i32 2
+; A320-NEXT: [[TMP67:%.*]] = insertelement <4 x double> [[TMP66]], double [[TMP63]], i32 3
+; A320-NEXT: [[TMP68:%.*]] = load double, ptr [[TMP37]], align 8
+; A320-NEXT: [[TMP69:%.*]] = load double, ptr [[TMP39]], align 8
+; A320-NEXT: [[TMP70:%.*]] = load double, ptr [[TMP41]], align 8
+; A320-NEXT: [[TMP71:%.*]] = load double, ptr [[TMP43]], align 8
+; A320-NEXT: [[TMP72:%.*]] = insertelement <4 x double> poison, double [[TMP68]], i32 0
+; A320-NEXT: [[TMP73:%.*]] = insertelement <4 x double> [[TMP72]], double [[TMP69]], i32 1
+; A320-NEXT: [[TMP74:%.*]] = insertelement <4 x double> [[TMP73]], double [[TMP70]], i32 2
+; A320-NEXT: [[TMP75:%.*]] = insertelement <4 x double> [[TMP74]], double [[TMP71]], i32 3
+; A320-NEXT: [[TMP76:%.*]] = load double, ptr [[TMP45]], align 8
+; A320-NEXT: [[TMP77:%.*]] = load double, ptr [[TMP47]], align 8
+; A320-NEXT: [[TMP78:%.*]] = load double, ptr [[TMP49]], align 8
+; A320-NEXT: [[TMP79:%.*]] = load double, ptr [[TMP51]], align 8
+; A320-NEXT: [[TMP80:%.*]] = insertelement <4 x double> poison, double [[TMP76]], i32 0
+; A320-NEXT: [[TMP81:%.*]] = insertelement <4 x double> [[TMP80]], double [[TMP77]], i32 1
+; A320-NEXT: [[TMP82:%.*]] = insertelement <4 x double> [[TMP81]], double [[TMP78]], i32 2
+; A320-NEXT: [[TMP83:%.*]] = insertelement <4 x double> [[TMP82]], double [[TMP79]], i32 3
+; A320-NEXT: [[TMP84:%.*]] = fmul fast <4 x double> [[TMP59]], [[WIDE_LOAD]]
+; A320-NEXT: [[TMP85:%.*]] = fmul fast <4 x double> [[TMP67]], [[WIDE_LOAD8]]
+; A320-NEXT: [[TMP86:%.*]] = fmul fast <4 x double> [[TMP75]], [[WIDE_LOAD9]]
+; A320-NEXT: [[TMP87:%.*]] = fmul fast <4 x double> [[TMP83]], [[WIDE_LOAD10]]
+; A320-NEXT: [[TMP88]] = fadd fast <4 x double> [[TMP84]], [[VEC_PHI]]
+; A320-NEXT: [[TMP89]] = fadd fast <4 x double> [[TMP85]], [[VEC_PHI4]]
+; A320-NEXT: [[TMP90]] = fadd fast <4 x double> [[TMP86]], [[VEC_PHI5]]
+; A320-NEXT: [[TMP91]] = fadd fast <4 x double> [[TMP87]], [[VEC_PHI6]]
+; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; A320-NEXT: [[TMP92:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; A320-NEXT: br i1 [[TMP92]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; A320: [[MIDDLE_BLOCK1]]:
+; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x double> [[TMP89]], [[TMP88]]
+; A320-NEXT: [[BIN_RDX15:%.*]] = fadd fast <4 x double> [[TMP90]], [[BIN_RDX]]
+; A320-NEXT: [[BIN_RDX16:%.*]] = fadd fast <4 x double> [[TMP91]], [[BIN_RDX15]]
+; A320-NEXT: [[TMP93:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[BIN_RDX16]])
+; A320-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; A320-NEXT: br i1 [[CMP_N1]], label %[[EXIT_INNER:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; A320: [[VEC_EPILOG_ITER_CHECK]]:
+; A320-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; A320-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; A320: [[VEC_EPILOG_PH]]:
+; A320-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP93]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_PH]] ]
+; A320-NEXT: [[N_MOD_VF18:%.*]] = urem i64 [[TMP3]], 4
+; A320-NEXT: [[N_VEC19:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF18]]
+; A320-NEXT: [[TMP94:%.*]] = shl i64 [[N_VEC19]], 2
+; A320-NEXT: [[TMP95:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP94]]
+; A320-NEXT: [[TMP96:%.*]] = shl i64 [[N_VEC19]], 3
+; A320-NEXT: [[TMP97:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP96]]
+; A320-NEXT: [[TMP98:%.*]] = insertelement <4 x double> zeroinitializer, double [[BC_MERGE_RDX]], i32 0
+; A320-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; A320: [[VEC_EPILOG_VECTOR_BODY]]:
+; A320-NEXT: [[INDEX20:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT26:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI21:%.*]] = phi <4 x double> [ [[TMP98]], %[[VEC_EPILOG_PH]] ], [ [[TMP119:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; A320-NEXT: [[TMP99:%.*]] = shl i64 [[INDEX20]], 2
+; A320-NEXT: [[NEXT_GEP22:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP99]]
+; A320-NEXT: [[TMP100:%.*]] = shl i64 [[INDEX20]], 3
+; A320-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP100]]
+; A320-NEXT: [[WIDE_LOAD24:%.*]] = load <4 x double>, ptr [[NEXT_GEP23]], align 8
+; A320-NEXT: [[WIDE_LOAD25:%.*]] = load <4 x i32>, ptr [[NEXT_GEP22]], align 4
+; A320-NEXT: [[TMP101:%.*]] = zext <4 x i32> [[WIDE_LOAD25]] to <4 x i64>
+; A320-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP101]], i64 0
; A320-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP14]]
+; A320-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP101]], i64 1
; A320-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP15]]
+; A320-NEXT: [[TMP106:%.*]] = extractelement <4 x i64> [[TMP101]], i64 2
+; A320-NEXT: [[TMP107:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP106]]
+; A320-NEXT: [[TMP108:%.*]] = extractelement <4 x i64> [[TMP101]], i64 3
+; A320-NEXT: [[TMP109:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP108]]
; A320-NEXT: [[TMP18:%.*]] = load double, ptr [[TMP16]], align 8
; A320-NEXT: [[TMP19:%.*]] = load double, ptr [[TMP17]], align 8
-; A320-NEXT: [[TMP20:%.*]] = fmul fast double [[TMP18]], [[TMP10]]
-; A320-NEXT: [[TMP21:%.*]] = fmul fast double [[TMP19]], [[TMP11]]
-; A320-NEXT: [[TMP22]] = fadd fast double [[TMP20]], [[VEC_PHI]]
-; A320-NEXT: [[TMP23]] = fadd fast double [[TMP21]], [[VEC_PHI5]]
-; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; A320-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; A320-NEXT: [[TMP112:%.*]] = load double, ptr [[TMP107]], align 8
+; A320-NEXT: [[TMP113:%.*]] = load double, ptr [[TMP109]], align 8
+; A320-NEXT: [[TMP114:%.*]] = insertelement <4 x double> poison, double [[TMP18]], i32 0
+; A320-NEXT: [[TMP115:%.*]] = insertelement <4 x double> [[TMP114]], double [[TMP19]], i32 1
+; A320-NEXT: [[TMP116:%.*]] = insertelement <4 x double> [[TMP115]], double [[TMP112]], i32 2
+; A320-NEXT: [[TMP117:%.*]] = insertelement <4 x double> [[TMP116]], double [[TMP113]], i32 3
+; A320-NEXT: [[TMP118:%.*]] = fmul fast <4 x double> [[TMP117]], [[WIDE_LOAD24]]
+; A320-NEXT: [[TMP119]] = fadd fast <4 x double> [[TMP118]], [[VEC_PHI21]]
+; A320-NEXT: [[INDEX_NEXT26]] = add nuw i64 [[INDEX20]], 4
+; A320-NEXT: [[TMP120:%.*]] = icmp eq i64 [[INDEX_NEXT26]], [[N_VEC19]]
+; A320-NEXT: br i1 [[TMP120]], label %[[MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
-; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast double [[TMP23]], [[TMP22]]
-; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER:.*]], label %[[SCALAR_PH]]
+; A320-NEXT: [[TMP121:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP119]])
+; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC19]]
+; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER]], label %[[SCALAR_PH]]
; A320: [[SCALAR_PH]]:
-; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TPM27]], %[[OUTER]] ]
-; A320-NEXT: [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END3]], %[[MIDDLE_BLOCK]] ], [ [[TPM32]], %[[OUTER]] ]
-; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[OUTER]] ]
+; A320-NEXT: [[BC_RESUME_VAL28:%.*]] = phi ptr [ [[TMP95]], %[[MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TPM27]], %[[OUTER]] ]
+; A320-NEXT: [[BC_RESUME_VAL29:%.*]] = phi ptr [ [[TMP97]], %[[MIDDLE_BLOCK]] ], [ [[IND_END3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TPM32]], %[[OUTER]] ]
+; A320-NEXT: [[BC_MERGE_RDX30:%.*]] = phi double [ [[TMP121]], %[[MIDDLE_BLOCK]] ], [ [[TMP93]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[OUTER]] ]
; A320-NEXT: br label %[[INNER:.*]]
; A320: [[INNER]]:
-; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ]
-; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL28]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL29]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX30]], %[[SCALAR_PH]] ]
; A320-NEXT: [[TPM44:%.*]] = load double, ptr [[PHI_PTR_F64]], align 8
; A320-NEXT: [[TPM45:%.*]] = load i32, ptr [[PHI_PTR_I32]], align 4
; A320-NEXT: [[TPM46:%.*]] = zext i32 [[TPM45]] to i64
@@ -90,9 +219,9 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[NEXT_I32]] = getelementptr inbounds i32, ptr [[PHI_PTR_I32]], i64 1
; A320-NEXT: [[NEXT_F64]] = getelementptr inbounds double, ptr [[PHI_PTR_F64]], i64 1
; A320-NEXT: [[DONE:%.*]] = icmp eq ptr [[NEXT_I32]], [[TPM29]]
-; A320-NEXT: br i1 [[DONE]], label %[[EXIT_INNER]], label %[[INNER]], !llvm.loop [[LOOP3:![0-9]+]]
+; A320-NEXT: br i1 [[DONE]], label %[[EXIT_INNER]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]]
; A320: [[EXIT_INNER]]:
-; A320-NEXT: [[TPM50_LCSSA:%.*]] = phi double [ [[TPM50]], %[[INNER]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
+; A320-NEXT: [[TPM50_LCSSA:%.*]] = phi double [ [[TPM50]], %[[INNER]] ], [ [[TMP93]], %[[MIDDLE_BLOCK1]] ], [ [[TMP121]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: [[TPM35:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 0
; A320-NEXT: [[TPM37:%.*]] = fsub fast double 0.000000e+00, [[TPM50_LCSSA]]
; A320-NEXT: store double [[TPM37]], ptr [[TPM35]], align 8
@@ -177,7 +306,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[TMP7]] = fadd fast <2 x double> [[VEC_PHI3]], [[WIDE_LOAD6]]
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP2]]
; A320-NEXT: [[BIN_RDX7:%.*]] = fadd fast <2 x double> [[TMP10]], [[BIN_RDX]]
@@ -197,7 +326,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[SUM_NEXT]] = fadd fast double [[SUM]], [[VAL]]
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
; A320-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi double [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
@@ -280,7 +409,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[TMP15]] = fadd fast <2 x double> [[VEC_PHI3]], [[TMP11]]
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP6]]
; A320-NEXT: [[BIN_RDX11:%.*]] = fadd fast <2 x double> [[TMP16]], [[BIN_RDX]]
@@ -303,7 +432,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[ACC_NEXT]] = fadd fast double [[ACC]], [[PROD]]
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
; A320-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi double [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP17]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
@@ -342,9 +471,11 @@ exit:
; A320: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; A320: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; A320: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; A320: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; A320: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
; A320: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; A320: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
; A320: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; A320: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; A320: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; A320: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index e46509a4e1c6e..e48aabed0e6a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -72,7 +72,15 @@ define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr n
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 123
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP12]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i32 2
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP10]], splat (i32 123)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP11]], i64 3
; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -106,29 +114,165 @@ define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias
; CHECK-LABEL: define void @replicating_load_used_as_store_addr_3(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[INVAR_DST:%.*]], i8 [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[TMP87:%.*]] = xor i8 [[X]], 10
+; CHECK-NEXT: [[TMP88:%.*]] = zext i8 [[TMP87]] to i64
+; CHECK-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP88]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP94:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP103:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP107:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP112:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP113:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = zext i8 [[TMP90]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = zext i8 [[TMP94]] to i32
+; CHECK-NEXT: [[TMP21:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = zext i8 [[TMP103]] to i32
+; CHECK-NEXT: [[TMP23:%.*]] = zext i8 [[TMP107]] to i32
+; CHECK-NEXT: [[TMP24:%.*]] = zext i8 [[TMP112]] to i32
+; CHECK-NEXT: [[TMP25:%.*]] = zext i8 [[TMP113]] to i32
+; CHECK-NEXT: [[TMP26:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT: [[TMP27:%.*]] = zext i8 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP28:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP29:%.*]] = zext i8 [[TMP13]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = zext i8 [[TMP16]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = zext i8 [[TMP17]] to i32
+; CHECK-NEXT: [[TMP34:%.*]] = zext i8 [[TMP18]] to i32
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <16 x i32> poison, i32 [[TMP19]], i32 0
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <16 x i32> [[TMP35]], i32 [[TMP20]], i32 1
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <16 x i32> [[TMP36]], i32 [[TMP21]], i32 2
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <16 x i32> [[TMP37]], i32 [[TMP22]], i32 3
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <16 x i32> [[TMP38]], i32 [[TMP23]], i32 4
+; CHECK-NEXT: [[TMP40:%.*]] = insertelement <16 x i32> [[TMP39]], i32 [[TMP24]], i32 5
+; CHECK-NEXT: [[TMP41:%.*]] = insertelement <16 x i32> [[TMP40]], i32 [[TMP25]], i32 6
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <16 x i32> [[TMP41]], i32 [[TMP26]], i32 7
+; CHECK-NEXT: [[TMP43:%.*]] = insertelement <16 x i32> [[TMP42]], i32 [[TMP27]], i32 8
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <16 x i32> [[TMP43]], i32 [[TMP28]], i32 9
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <16 x i32> [[TMP44]], i32 [[TMP29]], i32 10
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <16 x i32> [[TMP45]], i32 [[TMP30]], i32 11
+; CHECK-NEXT: [[TMP47:%.*]] = insertelement <16 x i32> [[TMP46]], i32 [[TMP31]], i32 12
+; CHECK-NEXT: [[TMP48:%.*]] = insertelement <16 x i32> [[TMP47]], i32 [[TMP32]], i32 13
+; CHECK-NEXT: [[TMP49:%.*]] = insertelement <16 x i32> [[TMP48]], i32 [[TMP33]], i32 14
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <16 x i32> [[TMP49]], i32 [[TMP34]], i32 15
+; CHECK-NEXT: [[TMP51:%.*]] = xor <16 x i32> [[TMP50]], splat (i32 111)
+; CHECK-NEXT: [[TMP52:%.*]] = zext i32 [[TMP19]] to i64
+; CHECK-NEXT: [[TMP53:%.*]] = zext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[TMP54:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT: [[TMP55:%.*]] = zext i32 [[TMP22]] to i64
+; CHECK-NEXT: [[TMP56:%.*]] = zext i32 [[TMP23]] to i64
+; CHECK-NEXT: [[TMP57:%.*]] = zext i32 [[TMP24]] to i64
+; CHECK-NEXT: [[TMP58:%.*]] = zext i32 [[TMP25]] to i64
+; CHECK-NEXT: [[TMP59:%.*]] = zext i32 [[TMP26]] to i64
+; CHECK-NEXT: [[TMP60:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT: [[TMP61:%.*]] = zext i32 [[TMP28]] to i64
+; CHECK-NEXT: [[TMP62:%.*]] = zext i32 [[TMP29]] to i64
+; CHECK-NEXT: [[TMP63:%.*]] = zext i32 [[TMP30]] to i64
+; CHECK-NEXT: [[TMP64:%.*]] = zext i32 [[TMP31]] to i64
+; CHECK-NEXT: [[TMP65:%.*]] = zext i32 [[TMP32]] to i64
+; CHECK-NEXT: [[TMP66:%.*]] = zext i32 [[TMP33]] to i64
+; CHECK-NEXT: [[TMP67:%.*]] = zext i32 [[TMP34]] to i64
+; CHECK-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]
+; CHECK-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP53]]
+; CHECK-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP54]]
+; CHECK-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP55]]
+; CHECK-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP57]]
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP58]]
+; CHECK-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP59]]
+; CHECK-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP60]]
+; CHECK-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP61]]
+; CHECK-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP62]]
+; CHECK-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP63]]
+; CHECK-NEXT: [[TMP80:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP64]]
+; CHECK-NEXT: [[TMP81:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP65]]
+; CHECK-NEXT: [[TMP82:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP66]]
+; CHECK-NEXT: [[TMP83:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP67]]
+; CHECK-NEXT: store i8 0, ptr [[TMP68]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP69]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP70]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP71]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP72]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP73]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP74]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP75]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP76]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP77]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP78]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP79]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP80]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP81]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP82]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP83]], align 1
+; CHECK-NEXT: [[TMP84:%.*]] = trunc <16 x i32> [[TMP51]] to <16 x i8>
+; CHECK-NEXT: [[TMP85:%.*]] = extractelement <16 x i8> [[TMP84]], i64 15
+; CHECK-NEXT: store i8 [[TMP85]], ptr [[INVAR_DST]], align 1
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 16
+; CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 96
+; CHECK-NEXT: br i1 [[TMP86]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK1]]:
+; CHECK-NEXT: br i1 false, [[EXIT1:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[TMP0:%.*]] = xor i8 [[X]], 10
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP91:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP92:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP93:%.*]] = load i8, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP4]], 111
+; CHECK-NEXT: [[TMP95:%.*]] = zext i8 [[TMP91]] to i32
+; CHECK-NEXT: [[TMP96:%.*]] = zext i8 [[TMP92]] to i32
+; CHECK-NEXT: [[TMP97:%.*]] = zext i8 [[TMP93]] to i32
+; CHECK-NEXT: [[TMP98:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP99:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP95]], i32 1
+; CHECK-NEXT: [[TMP100:%.*]] = insertelement <4 x i32> [[TMP99]], i32 [[TMP96]], i32 2
+; CHECK-NEXT: [[TMP101:%.*]] = insertelement <4 x i32> [[TMP100]], i32 [[TMP97]], i32 3
+; CHECK-NEXT: [[TMP102:%.*]] = xor <4 x i32> [[TMP101]], splat (i32 111)
; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP104:%.*]] = zext i32 [[TMP95]] to i64
+; CHECK-NEXT: [[TMP105:%.*]] = zext i32 [[TMP96]] to i64
+; CHECK-NEXT: [[TMP106:%.*]] = zext i32 [[TMP97]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP108:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP104]]
+; CHECK-NEXT: [[TMP109:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP105]]
+; CHECK-NEXT: [[TMP110:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP106]]
; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: store i8 0, ptr [[TMP108]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP109]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP110]], align 1
+; CHECK-NEXT: [[TMP111:%.*]] = trunc <4 x i32> [[TMP102]] to <4 x i8>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP111]], i64 3
; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1]], label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -201,7 +345,7 @@ define void @uniform_gep_for_replicating_gep(ptr %dst) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD_3]], splat (i32 2)
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -255,7 +399,7 @@ define void @test_load_gep_widen_induction(ptr noalias %dst, ptr noalias %dst2)
; CHECK-NEXT: store ptr [[TMP42]], ptr [[TMP10]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -318,7 +462,7 @@ define ptr @replicating_store_in_conditional_latch(ptr %p, i32 %n) {
; CHECK-NEXT: store ptr null, ptr [[TMP15]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -483,21 +627,21 @@ define void @test_prefer_vector_addressing(ptr %start, ptr %ms, ptr noalias %src
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA11:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA11]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA11]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA11]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA14:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA14]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA14]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA14]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA16:![0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA16]]
-; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA16]]
-; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA16]]
+; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA19]]
+; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA19]]
+; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA19]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -750,7 +894,7 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index 3c4833012a04c..7ce0c3a5c5ed6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -72,7 +72,15 @@ define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr n
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 123
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP12]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i32 2
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP10]], splat (i32 123)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP11]], i64 3
; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -106,29 +114,165 @@ define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias
; CHECK-LABEL: define void @replicating_load_used_as_store_addr_3(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[INVAR_DST:%.*]], i8 [[X:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[TMP87:%.*]] = xor i8 [[X]], 10
+; CHECK-NEXT: [[TMP88:%.*]] = zext i8 [[TMP87]] to i64
+; CHECK-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP88]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP90:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP94:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP103:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP107:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP112:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP113:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP89]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = zext i8 [[TMP90]] to i32
+; CHECK-NEXT: [[TMP20:%.*]] = zext i8 [[TMP94]] to i32
+; CHECK-NEXT: [[TMP21:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = zext i8 [[TMP103]] to i32
+; CHECK-NEXT: [[TMP23:%.*]] = zext i8 [[TMP107]] to i32
+; CHECK-NEXT: [[TMP24:%.*]] = zext i8 [[TMP112]] to i32
+; CHECK-NEXT: [[TMP25:%.*]] = zext i8 [[TMP113]] to i32
+; CHECK-NEXT: [[TMP26:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT: [[TMP27:%.*]] = zext i8 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP28:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP29:%.*]] = zext i8 [[TMP13]] to i32
+; CHECK-NEXT: [[TMP30:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = zext i8 [[TMP16]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = zext i8 [[TMP17]] to i32
+; CHECK-NEXT: [[TMP34:%.*]] = zext i8 [[TMP18]] to i32
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <16 x i32> poison, i32 [[TMP19]], i32 0
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <16 x i32> [[TMP35]], i32 [[TMP20]], i32 1
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <16 x i32> [[TMP36]], i32 [[TMP21]], i32 2
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <16 x i32> [[TMP37]], i32 [[TMP22]], i32 3
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <16 x i32> [[TMP38]], i32 [[TMP23]], i32 4
+; CHECK-NEXT: [[TMP40:%.*]] = insertelement <16 x i32> [[TMP39]], i32 [[TMP24]], i32 5
+; CHECK-NEXT: [[TMP41:%.*]] = insertelement <16 x i32> [[TMP40]], i32 [[TMP25]], i32 6
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <16 x i32> [[TMP41]], i32 [[TMP26]], i32 7
+; CHECK-NEXT: [[TMP43:%.*]] = insertelement <16 x i32> [[TMP42]], i32 [[TMP27]], i32 8
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <16 x i32> [[TMP43]], i32 [[TMP28]], i32 9
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <16 x i32> [[TMP44]], i32 [[TMP29]], i32 10
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <16 x i32> [[TMP45]], i32 [[TMP30]], i32 11
+; CHECK-NEXT: [[TMP47:%.*]] = insertelement <16 x i32> [[TMP46]], i32 [[TMP31]], i32 12
+; CHECK-NEXT: [[TMP48:%.*]] = insertelement <16 x i32> [[TMP47]], i32 [[TMP32]], i32 13
+; CHECK-NEXT: [[TMP49:%.*]] = insertelement <16 x i32> [[TMP48]], i32 [[TMP33]], i32 14
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <16 x i32> [[TMP49]], i32 [[TMP34]], i32 15
+; CHECK-NEXT: [[TMP51:%.*]] = xor <16 x i32> [[TMP50]], splat (i32 111)
+; CHECK-NEXT: [[TMP52:%.*]] = zext i32 [[TMP19]] to i64
+; CHECK-NEXT: [[TMP53:%.*]] = zext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[TMP54:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT: [[TMP55:%.*]] = zext i32 [[TMP22]] to i64
+; CHECK-NEXT: [[TMP56:%.*]] = zext i32 [[TMP23]] to i64
+; CHECK-NEXT: [[TMP57:%.*]] = zext i32 [[TMP24]] to i64
+; CHECK-NEXT: [[TMP58:%.*]] = zext i32 [[TMP25]] to i64
+; CHECK-NEXT: [[TMP59:%.*]] = zext i32 [[TMP26]] to i64
+; CHECK-NEXT: [[TMP60:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT: [[TMP61:%.*]] = zext i32 [[TMP28]] to i64
+; CHECK-NEXT: [[TMP62:%.*]] = zext i32 [[TMP29]] to i64
+; CHECK-NEXT: [[TMP63:%.*]] = zext i32 [[TMP30]] to i64
+; CHECK-NEXT: [[TMP64:%.*]] = zext i32 [[TMP31]] to i64
+; CHECK-NEXT: [[TMP65:%.*]] = zext i32 [[TMP32]] to i64
+; CHECK-NEXT: [[TMP66:%.*]] = zext i32 [[TMP33]] to i64
+; CHECK-NEXT: [[TMP67:%.*]] = zext i32 [[TMP34]] to i64
+; CHECK-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]
+; CHECK-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP53]]
+; CHECK-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP54]]
+; CHECK-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP55]]
+; CHECK-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP56]]
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP57]]
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP58]]
+; CHECK-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP59]]
+; CHECK-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP60]]
+; CHECK-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP61]]
+; CHECK-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP62]]
+; CHECK-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP63]]
+; CHECK-NEXT: [[TMP80:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP64]]
+; CHECK-NEXT: [[TMP81:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP65]]
+; CHECK-NEXT: [[TMP82:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP66]]
+; CHECK-NEXT: [[TMP83:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP67]]
+; CHECK-NEXT: store i8 0, ptr [[TMP68]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP69]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP70]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP71]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP72]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP73]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP74]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP75]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP76]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP77]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP78]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP79]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP80]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP81]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP82]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP83]], align 1
+; CHECK-NEXT: [[TMP84:%.*]] = trunc <16 x i32> [[TMP51]] to <16 x i8>
+; CHECK-NEXT: [[TMP85:%.*]] = extractelement <16 x i8> [[TMP84]], i64 15
+; CHECK-NEXT: store i8 [[TMP85]], ptr [[INVAR_DST]], align 1
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 16
+; CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 96
+; CHECK-NEXT: br i1 [[TMP86]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK1]]:
+; CHECK-NEXT: br i1 false, [[EXIT1:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[TMP0:%.*]] = xor i8 [[X]], 10
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP91:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP92:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP93:%.*]] = load i8, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP4]], 111
+; CHECK-NEXT: [[TMP95:%.*]] = zext i8 [[TMP91]] to i32
+; CHECK-NEXT: [[TMP96:%.*]] = zext i8 [[TMP92]] to i32
+; CHECK-NEXT: [[TMP97:%.*]] = zext i8 [[TMP93]] to i32
+; CHECK-NEXT: [[TMP98:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP99:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP95]], i32 1
+; CHECK-NEXT: [[TMP100:%.*]] = insertelement <4 x i32> [[TMP99]], i32 [[TMP96]], i32 2
+; CHECK-NEXT: [[TMP101:%.*]] = insertelement <4 x i32> [[TMP100]], i32 [[TMP97]], i32 3
+; CHECK-NEXT: [[TMP102:%.*]] = xor <4 x i32> [[TMP101]], splat (i32 111)
; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP104:%.*]] = zext i32 [[TMP95]] to i64
+; CHECK-NEXT: [[TMP105:%.*]] = zext i32 [[TMP96]] to i64
+; CHECK-NEXT: [[TMP106:%.*]] = zext i32 [[TMP97]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP108:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP104]]
+; CHECK-NEXT: [[TMP109:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP105]]
+; CHECK-NEXT: [[TMP110:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP106]]
; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: store i8 0, ptr [[TMP108]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP109]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP110]], align 1
+; CHECK-NEXT: [[TMP111:%.*]] = trunc <4 x i32> [[TMP102]] to <4 x i8>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP111]], i64 3
; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1]], label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -183,7 +327,7 @@ define void @uniform_gep_for_replicating_gep(ptr %dst) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2)
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -251,7 +395,7 @@ define void @test_load_gep_widen_induction(ptr noalias %dst, ptr noalias %dst2)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD_3]], splat (i64 2)
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -314,7 +458,7 @@ define ptr @replicating_store_in_conditional_latch(ptr %p, i32 %n) #0 {
; CHECK-NEXT: store ptr null, ptr [[TMP15]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -479,21 +623,21 @@ define void @test_prefer_vector_addressing(ptr %start, ptr %ms, ptr noalias %src
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA12:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA15:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA15]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA15]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA17:![0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA17]]
+; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA20:![0-9]+]]
+; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA20]]
+; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA20]]
+; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA20]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -746,7 +890,7 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
index c7b1bcdf31e45..4a50e0b05d7c5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -516,29 +516,48 @@ exit:
define void @interleave_group_with_gather(ptr %indices, ptr %src, i64 %n) {
; CHECK-LABEL: define void @interleave_group_with_gather(
; CHECK-SAME: ptr [[INDICES:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[OUT_GEP:%.*]] = getelementptr { double, double }, ptr null, i64 [[IV]]
; CHECK-NEXT: [[IDX_GEP:%.*]] = getelementptr i32, ptr [[INDICES]], i64 [[IV]]
-; CHECK-NEXT: [[IDX:%.*]] = load i32, ptr [[IDX_GEP]], align 4
-; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[IDX]] to i64
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[IDX_GEP]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[IDX_EXT:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT: [[OUT_M1_0_GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT: [[OUT_M1_1_GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP10]]
; CHECK-NEXT: [[SRC_VAL:%.*]] = load double, ptr [[SRC_GEP]], align 8
-; CHECK-NEXT: [[OUT_M1_0_GEP:%.*]] = getelementptr i8, ptr [[OUT_GEP]], i64 -16
; CHECK-NEXT: [[OUT_M1_0:%.*]] = load double, ptr [[OUT_M1_0_GEP]], align 8
-; CHECK-NEXT: [[ADD_0:%.*]] = fadd double 1.000000e+01, [[SRC_VAL]]
-; CHECK-NEXT: store double [[ADD_0]], ptr [[OUT_M1_0_GEP]], align 8
-; CHECK-NEXT: [[OUT_M1_1_GEP:%.*]] = getelementptr i8, ptr [[OUT_GEP]], i64 -8
; CHECK-NEXT: [[OUT_M1_1:%.*]] = load double, ptr [[OUT_M1_1_GEP]], align 8
-; CHECK-NEXT: [[ADD_1:%.*]] = fadd double 1.000000e+01, [[SRC_VAL]]
-; CHECK-NEXT: store double [[ADD_1]], ptr [[OUT_M1_1_GEP]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> poison, double [[SRC_VAL]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[OUT_M1_0]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[OUT_M1_1]], i32 2
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 3
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[OUT_GEP]], i64 -16
+; CHECK-NEXT: [[TMP21:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[TMP19]]
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP21]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP22]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/ARM/replicating-load-store-costs.ll
index fd83a012541b5..ab92e07a9f246 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/replicating-load-store-costs.ll
@@ -7,9 +7,85 @@ define void @replicating_load_used_by_other_load(i32 %arg, ptr %a, i32 %b) {
; CHECK-LABEL: define void @replicating_load_used_by_other_load(
; CHECK-SAME: i32 [[ARG:%.*]], ptr [[A:%.*]], i32 [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 100, [[ARG]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[ARG]], [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[ARG]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[ARG]], %[[ENTRY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[VEC_IND]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 1)
+; CHECK-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[TMP5:%.*]] = shl <4 x i32> [[VEC_IND]], splat (i32 1)
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP5]], splat (i32 2)
+; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i32> [[TMP6]], [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i32> [[BROADCAST_SPLAT]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i32> [[TMP9]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP11:%.*]] = lshr <4 x i32> [[TMP4]], splat (i32 1)
+; CHECK-NEXT: [[TMP12:%.*]] = xor <4 x i32> [[TMP2]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP13:%.*]] = and <4 x i32> [[TMP12]], splat (i32 1)
+; CHECK-NEXT: [[TMP14:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 2147483646)
+; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 254)
+; CHECK-NEXT: [[TMP17:%.*]] = shl <4 x i32> [[TMP15]], splat (i32 1)
+; CHECK-NEXT: [[TMP18:%.*]] = xor <4 x i32> [[TMP17]], splat (i32 2)
+; CHECK-NEXT: [[TMP19:%.*]] = or <4 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = xor <4 x i32> [[TMP11]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = xor <4 x i32> [[TMP20]], [[TMP10]]
+; CHECK-NEXT: [[TMP22:%.*]] = and <4 x i32> [[TMP21]], splat (i32 255)
+; CHECK-NEXT: [[TMP23:%.*]] = xor <4 x i32> [[TMP22]], splat (i32 1)
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP23]], i64 1
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP23]], i64 2
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP23]], i64 3
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[A]], i32 [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP27]], align 1
+; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP29]], align 1
+; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP31]], align 1
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i8> poison, i8 [[TMP32]], i32 0
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i8> [[TMP36]], i8 [[TMP33]], i32 1
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i8> [[TMP37]], i8 [[TMP34]], i32 2
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x i8> [[TMP38]], i8 [[TMP35]], i32 3
+; CHECK-NEXT: [[TMP40:%.*]] = zext <4 x i8> [[TMP39]] to <4 x i32>
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i32> [[TMP40]], i64 0
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr null, i32 [[TMP41]]
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <4 x i32> [[TMP40]], i64 1
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr null, i32 [[TMP43]]
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x i32> [[TMP40]], i64 2
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr null, i32 [[TMP45]]
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i32> [[TMP40]], i64 3
+; CHECK-NEXT: [[TMP48:%.*]] = getelementptr i32, ptr null, i32 [[TMP47]]
+; CHECK-NEXT: store i32 0, ptr [[TMP42]], align 4
+; CHECK-NEXT: store i32 0, ptr [[TMP44]], align 4
+; CHECK-NEXT: store i32 0, ptr [[TMP46]], align 4
+; CHECK-NEXT: store i32 0, ptr [[TMP48]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP49:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[ARG]], %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[IV]], 1
; CHECK-NEXT: [[AND_1:%.*]] = and i32 [[IV]], 1
; CHECK-NEXT: [[SHL_1:%.*]] = shl i32 [[IV]], 2
@@ -39,7 +115,7 @@ define void @replicating_load_used_by_other_load(i32 %arg, ptr %a, i32 %b) {
; CHECK-NEXT: store i32 0, ptr [[GEP_2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -82,3 +158,9 @@ loop:
exit:
ret void
}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
index 1ac8f11262064..3daf40126f02d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i16-with-i8-index.ll
@@ -18,18 +18,18 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE-LABEL: 'test'
; SSE: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i16, ptr %inB, align 2
-; SSE: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE: Cost of 192 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE: Cost of 32 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i16, ptr %inB, align 2
-; AVX1: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 193 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 386 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 33 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 66 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX2-SLOWGATHER-LABEL: 'test'
; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i16, ptr %inB, align 2
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
index 2d5a30019bacd..1f66ae25a7c9e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i32-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i32, ptr %inB, align 4
-; SSE2: Cost of 25 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 51 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 102 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 204 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 5 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 11 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 22 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 44 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i32, ptr %inB, align 4
-; SSE42: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 192 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 32 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i32, ptr %inB, align 4
-; AVX1: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 97 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 194 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 388 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 17 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 34 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 68 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX2-SLOWGATHER-LABEL: 'test'
; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i32, ptr %inB, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
index ce5828a46eea7..487f4398fade5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i64-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i64, ptr %inB, align 8
-; SSE2: Cost of 25 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 50 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 100 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 200 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 5 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 10 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 20 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 40 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i64, ptr %inB, align 8
-; SSE42: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 192 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 32 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i64, ptr %inB, align 8
-; AVX1: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 49 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 98 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 196 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 392 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 9 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 18 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 36 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 72 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX2-SLOWGATHER-LABEL: 'test'
; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i64, ptr %inB, align 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
index d894cdb753bcd..a964141f395e1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/gather-i8-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i8, ptr %inB, align 1
-; SSE2: Cost of 25 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 51 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 103 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE2: Cost of 207 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 5 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 11 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 23 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE2: Cost of 47 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i8, ptr %inB, align 1
-; SSE42: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; SSE42: Cost of 192 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; SSE42: Cost of 32 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i8, ptr %inB, align 1
-; AVX1: Cost of 24 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 48 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 96 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 192 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
-; AVX1: Cost of 385 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 4 for VF 2: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 8 for VF 4: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 16 for VF 8: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 32 for VF 16: REPLICATE ir<%valB> = load ir<%inB>
+; AVX1: Cost of 65 for VF 32: REPLICATE ir<%valB> = load ir<%inB>
;
; AVX2-SLOWGATHER-LABEL: 'test'
; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB = load i8, ptr %inB, align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
index 6782bbfeb53b3..00e0b2a58f08d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i16-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, ptr %out, align 2
-; SSE2: Cost of 28 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 56 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 112 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 224 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 8 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 16 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 32 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 64 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, ptr %out, align 2
-; SSE42: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 52 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 104 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 208 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 12 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 24 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 48 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, ptr %out, align 2
-; AVX1: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 53 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 106 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 213 for VF 16: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 426 for VF 32: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 13 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 26 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 53 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 106 for VF 32: REPLICATE store ir<%valB>, ir<%out>
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, ptr %out, align 2
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
index 5e0b3277dd5e9..1e3db35b0bad2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i32-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
-; SSE2: Cost of 29 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 59 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 118 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 236 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 9 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 19 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 38 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 76 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
-; SSE42: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 52 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 104 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 208 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 12 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 24 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 48 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
-; AVX1: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 53 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 107 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 214 for VF 16: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 428 for VF 32: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 13 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 27 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 54 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 108 for VF 32: REPLICATE store ir<%valB>, ir<%out>
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
index fcbf6042dec14..a7f7da0cc243d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i64-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
-; SSE2: Cost of 29 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 58 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 116 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 232 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 9 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 18 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 36 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 72 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
-; SSE42: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 52 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 104 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 208 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 12 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 24 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 48 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
-; AVX1: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 54 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 108 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 216 for VF 16: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 432 for VF 32: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 14 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 28 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 56 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 112 for VF 32: REPLICATE store ir<%valB>, ir<%out>
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
index 2946cd291d7fa..2bb5855f41026 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/scatter-i8-with-i8-index.ll
@@ -18,25 +18,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test() {
; SSE2-LABEL: 'test'
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, ptr %out, align 1
-; SSE2: Cost of 29 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 59 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 119 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE2: Cost of 239 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 9 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 19 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 39 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE2: Cost of 79 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; SSE42-LABEL: 'test'
; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, ptr %out, align 1
-; SSE42: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 52 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 104 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; SSE42: Cost of 208 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 12 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 24 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; SSE42: Cost of 48 for VF 16: REPLICATE store ir<%valB>, ir<%out>
;
; AVX1-LABEL: 'test'
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, ptr %out, align 1
-; AVX1: Cost of 26 for VF 2: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 53 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 106 for VF 8: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 212 for VF 16: REPLICATE store ir<%valB>, ir<%out>
-; AVX1: Cost of 425 for VF 32: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 6 for VF 2: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 13 for VF 4: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 26 for VF 8: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 52 for VF 16: REPLICATE store ir<%valB>, ir<%out>
+; AVX1: Cost of 105 for VF 32: REPLICATE store ir<%valB>, ir<%out>
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, ptr %out, align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 1bfe36f9f1e77..e71f752e110ec 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -38,11 +38,11 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX1-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX1: [[VECTOR_BODY]]:
; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; AVX1-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
; AVX1-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
-; AVX1-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[TMP3]], <8 x i32> poison)
+; AVX1-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP9]], <8 x i1> [[TMP3]], <8 x i32> poison)
; AVX1-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
; AVX1-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; AVX1-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP5]], ptr align 4 [[TMP6]], <8 x i1> [[TMP3]])
@@ -75,13 +75,13 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX2: [[VECTOR_BODY]]:
; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 8
-; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 24
-; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; AVX2-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 8
+; AVX2-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 16
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 24
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP29]], align 4
; AVX2-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
+; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP30]], align 4
; AVX2-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
; AVX2-NEXT: [[TMP6:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX2-NEXT: [[TMP7:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -156,13 +156,13 @@ define void @foo1(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 32
-; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 48
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
+; AVX512-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 16
+; AVX512-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 32
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP29]], i64 48
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP29]], align 4
; AVX512-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i32>, ptr [[TMP3]], align 4
-; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
+; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr [[TMP30]], align 4
; AVX512-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP5]], align 4
; AVX512-NEXT: [[TMP6:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX512-NEXT: [[TMP7:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -265,11 +265,11 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
; AVX1-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX1: [[VECTOR_BODY]]:
; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP2]], align 4
+; AVX1-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP8]], align 4
; AVX1-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
-; AVX1-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr addrspace(1) [[B]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 4 [[TMP4]], <8 x i1> [[TMP3]], <8 x i32> poison)
+; AVX1-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr addrspace(1) [[B]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 4 [[TMP9]], <8 x i1> [[TMP3]], <8 x i32> poison)
; AVX1-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
; AVX1-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(1) [[A]], i64 [[INDEX]]
; AVX1-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> [[TMP5]], ptr addrspace(1) align 4 [[TMP6]], <8 x i1> [[TMP3]])
@@ -302,13 +302,13 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX2: [[VECTOR_BODY]]:
; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
-; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 8
-; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 16
-; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 24
-; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP2]], align 4
+; AVX2-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 8
+; AVX2-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 16
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 24
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP29]], align 4
; AVX2-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP3]], align 4
-; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP4]], align 4
+; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP30]], align 4
; AVX2-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr addrspace(1) [[TMP5]], align 4
; AVX2-NEXT: [[TMP6:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX2-NEXT: [[TMP7:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -383,13 +383,13 @@ define void @foo1_addrspace1(ptr addrspace(1) nocapture %A, ptr addrspace(1) noc
; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 16
-; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 32
-; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP2]], i64 48
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP2]], align 4
+; AVX512-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 16
+; AVX512-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 32
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[TMP29]], i64 48
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP29]], align 4
; AVX512-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP3]], align 4
-; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP4]], align 4
+; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP30]], align 4
; AVX512-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr addrspace(1) [[TMP5]], align 4
; AVX512-NEXT: [[TMP6:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX512-NEXT: [[TMP7:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -501,11 +501,11 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX1-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX1: [[VECTOR_BODY]]:
; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; AVX1-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
; AVX1-NEXT: [[TMP3:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
-; AVX1-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[B]], i64 [[INDEX]]
-; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP4]], <8 x i1> [[TMP3]], <8 x float> poison)
+; AVX1-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[B]], i64 [[INDEX]]
+; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP10]], <8 x i1> [[TMP3]], <8 x float> poison)
; AVX1-NEXT: [[TMP5:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x float>
; AVX1-NEXT: [[TMP6:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD]], [[TMP5]]
; AVX1-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
@@ -539,13 +539,13 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX2: [[VECTOR_BODY]]:
; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 8
-; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 24
-; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; AVX2-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 8
+; AVX2-NEXT: [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 16
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 24
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP34]], align 4
; AVX2-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
+; AVX2-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP35]], align 4
; AVX2-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
; AVX2-NEXT: [[TMP6:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX2-NEXT: [[TMP7:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -625,13 +625,13 @@ define void @foo2(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX512: [[VECTOR_BODY]]:
; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
-; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 16
-; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 32
-; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 48
-; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP2]], align 4
+; AVX512-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 16
+; AVX512-NEXT: [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 32
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 48
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP34]], align 4
; AVX512-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i32>, ptr [[TMP3]], align 4
-; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
+; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i32>, ptr [[TMP35]], align 4
; AVX512-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP5]], align 4
; AVX512-NEXT: [[TMP6:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], splat (i32 100)
; AVX512-NEXT: [[TMP7:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD5]], splat (i32 100)
@@ -1974,23 +1974,180 @@ for.end:
define i32 @reverse_gather(ptr %p) {
; AVX1-LABEL: define i32 @reverse_gather(
; AVX1-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; AVX1-NEXT: [[ENTRY:.*]]:
+; AVX1-NEXT: [[ENTRY:.*:]]
+; AVX1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX1: [[VECTOR_PH]]:
; AVX1-NEXT: br label %[[LOOP:.*]]
; AVX1: [[LOOP]]:
-; AVX1-NEXT: [[ANY:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; AVX1-NEXT: [[IV:%.*]] = phi i32 [ 100, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; AVX1-NEXT: [[IV_NEXT]] = add i32 [[IV]], -1
+; AVX1-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; AVX1-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP100:%.*]], %[[LOOP]] ]
+; AVX1-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP101:%.*]], %[[LOOP]] ]
+; AVX1-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP102:%.*]], %[[LOOP]] ]
+; AVX1-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP103:%.*]], %[[LOOP]] ]
+; AVX1-NEXT: [[IV:%.*]] = sub i32 100, [[INDEX]]
+; AVX1-NEXT: [[IV_NEXT:%.*]] = add i32 [[IV]], -1
+; AVX1-NEXT: [[TMP2:%.*]] = add i32 [[IV]], -2
+; AVX1-NEXT: [[TMP3:%.*]] = add i32 [[IV]], -3
+; AVX1-NEXT: [[TMP4:%.*]] = add i32 [[IV]], -4
+; AVX1-NEXT: [[TMP5:%.*]] = add i32 [[IV]], -5
+; AVX1-NEXT: [[TMP6:%.*]] = add i32 [[IV]], -6
+; AVX1-NEXT: [[TMP7:%.*]] = add i32 [[IV]], -7
+; AVX1-NEXT: [[TMP8:%.*]] = add i32 [[IV]], -8
+; AVX1-NEXT: [[TMP9:%.*]] = add i32 [[IV]], -9
+; AVX1-NEXT: [[TMP10:%.*]] = add i32 [[IV]], -10
+; AVX1-NEXT: [[TMP11:%.*]] = add i32 [[IV]], -11
+; AVX1-NEXT: [[TMP12:%.*]] = add i32 [[IV]], -12
+; AVX1-NEXT: [[TMP13:%.*]] = add i32 [[IV]], -13
+; AVX1-NEXT: [[TMP14:%.*]] = add i32 [[IV]], -14
+; AVX1-NEXT: [[TMP15:%.*]] = add i32 [[IV]], -15
; AVX1-NEXT: [[IDX:%.*]] = zext i32 [[IV]] to i64
+; AVX1-NEXT: [[TMP17:%.*]] = zext i32 [[IV_NEXT]] to i64
+; AVX1-NEXT: [[TMP18:%.*]] = zext i32 [[TMP2]] to i64
+; AVX1-NEXT: [[TMP19:%.*]] = zext i32 [[TMP3]] to i64
+; AVX1-NEXT: [[TMP20:%.*]] = zext i32 [[TMP4]] to i64
+; AVX1-NEXT: [[TMP21:%.*]] = zext i32 [[TMP5]] to i64
+; AVX1-NEXT: [[TMP22:%.*]] = zext i32 [[TMP6]] to i64
+; AVX1-NEXT: [[TMP23:%.*]] = zext i32 [[TMP7]] to i64
+; AVX1-NEXT: [[TMP24:%.*]] = zext i32 [[TMP8]] to i64
+; AVX1-NEXT: [[TMP25:%.*]] = zext i32 [[TMP9]] to i64
+; AVX1-NEXT: [[TMP26:%.*]] = zext i32 [[TMP10]] to i64
+; AVX1-NEXT: [[TMP27:%.*]] = zext i32 [[TMP11]] to i64
+; AVX1-NEXT: [[TMP28:%.*]] = zext i32 [[TMP12]] to i64
+; AVX1-NEXT: [[TMP29:%.*]] = zext i32 [[TMP13]] to i64
+; AVX1-NEXT: [[TMP30:%.*]] = zext i32 [[TMP14]] to i64
+; AVX1-NEXT: [[TMP31:%.*]] = zext i32 [[TMP15]] to i64
; AVX1-NEXT: [[GEP:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[IDX]]
+; AVX1-NEXT: [[TMP33:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP17]]
+; AVX1-NEXT: [[TMP34:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP18]]
+; AVX1-NEXT: [[TMP35:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP19]]
+; AVX1-NEXT: [[TMP36:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP20]]
+; AVX1-NEXT: [[TMP37:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP21]]
+; AVX1-NEXT: [[TMP38:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP22]]
+; AVX1-NEXT: [[TMP39:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP23]]
+; AVX1-NEXT: [[TMP40:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP24]]
+; AVX1-NEXT: [[TMP41:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP25]]
+; AVX1-NEXT: [[TMP42:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP26]]
+; AVX1-NEXT: [[TMP43:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP27]]
+; AVX1-NEXT: [[TMP44:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP28]]
+; AVX1-NEXT: [[TMP45:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP29]]
+; AVX1-NEXT: [[TMP46:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP30]]
+; AVX1-NEXT: [[TMP47:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP31]]
; AVX1-NEXT: [[PTR:%.*]] = load ptr, ptr [[GEP]], align 8
+; AVX1-NEXT: [[TMP49:%.*]] = load ptr, ptr [[TMP33]], align 8
+; AVX1-NEXT: [[TMP50:%.*]] = load ptr, ptr [[TMP34]], align 8
+; AVX1-NEXT: [[TMP51:%.*]] = load ptr, ptr [[TMP35]], align 8
+; AVX1-NEXT: [[TMP52:%.*]] = load ptr, ptr [[TMP36]], align 8
+; AVX1-NEXT: [[TMP53:%.*]] = load ptr, ptr [[TMP37]], align 8
+; AVX1-NEXT: [[TMP54:%.*]] = load ptr, ptr [[TMP38]], align 8
+; AVX1-NEXT: [[TMP55:%.*]] = load ptr, ptr [[TMP39]], align 8
+; AVX1-NEXT: [[TMP56:%.*]] = load ptr, ptr [[TMP40]], align 8
+; AVX1-NEXT: [[TMP57:%.*]] = load ptr, ptr [[TMP41]], align 8
+; AVX1-NEXT: [[TMP58:%.*]] = load ptr, ptr [[TMP42]], align 8
+; AVX1-NEXT: [[TMP59:%.*]] = load ptr, ptr [[TMP43]], align 8
+; AVX1-NEXT: [[TMP60:%.*]] = load ptr, ptr [[TMP44]], align 8
+; AVX1-NEXT: [[TMP61:%.*]] = load ptr, ptr [[TMP45]], align 8
+; AVX1-NEXT: [[TMP62:%.*]] = load ptr, ptr [[TMP46]], align 8
+; AVX1-NEXT: [[TMP63:%.*]] = load ptr, ptr [[TMP47]], align 8
; AVX1-NEXT: [[V:%.*]] = load i32, ptr [[PTR]], align 8
-; AVX1-NEXT: [[NZ:%.*]] = icmp eq i32 [[V]], 0
-; AVX1-NEXT: [[SEL]] = select i1 [[NZ]], i32 [[ANY]], i32 0
-; AVX1-NEXT: [[COND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0
-; AVX1-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; AVX1-NEXT: [[TMP65:%.*]] = load i32, ptr [[TMP49]], align 8
+; AVX1-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP50]], align 8
+; AVX1-NEXT: [[TMP67:%.*]] = load i32, ptr [[TMP51]], align 8
+; AVX1-NEXT: [[TMP68:%.*]] = insertelement <4 x i32> poison, i32 [[V]], i32 0
+; AVX1-NEXT: [[TMP69:%.*]] = insertelement <4 x i32> [[TMP68]], i32 [[TMP65]], i32 1
+; AVX1-NEXT: [[TMP70:%.*]] = insertelement <4 x i32> [[TMP69]], i32 [[TMP66]], i32 2
+; AVX1-NEXT: [[TMP71:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP67]], i32 3
+; AVX1-NEXT: [[TMP72:%.*]] = load i32, ptr [[TMP52]], align 8
+; AVX1-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP53]], align 8
+; AVX1-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP54]], align 8
+; AVX1-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP55]], align 8
+; AVX1-NEXT: [[TMP76:%.*]] = insertelement <4 x i32> poison, i32 [[TMP72]], i32 0
+; AVX1-NEXT: [[TMP77:%.*]] = insertelement <4 x i32> [[TMP76]], i32 [[TMP73]], i32 1
+; AVX1-NEXT: [[TMP78:%.*]] = insertelement <4 x i32> [[TMP77]], i32 [[TMP74]], i32 2
+; AVX1-NEXT: [[TMP79:%.*]] = insertelement <4 x i32> [[TMP78]], i32 [[TMP75]], i32 3
+; AVX1-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP56]], align 8
+; AVX1-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP57]], align 8
+; AVX1-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP58]], align 8
+; AVX1-NEXT: [[TMP83:%.*]] = load i32, ptr [[TMP59]], align 8
+; AVX1-NEXT: [[TMP84:%.*]] = insertelement <4 x i32> poison, i32 [[TMP80]], i32 0
+; AVX1-NEXT: [[TMP85:%.*]] = insertelement <4 x i32> [[TMP84]], i32 [[TMP81]], i32 1
+; AVX1-NEXT: [[TMP86:%.*]] = insertelement <4 x i32> [[TMP85]], i32 [[TMP82]], i32 2
+; AVX1-NEXT: [[TMP87:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP83]], i32 3
+; AVX1-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP60]], align 8
+; AVX1-NEXT: [[TMP89:%.*]] = load i32, ptr [[TMP61]], align 8
+; AVX1-NEXT: [[TMP90:%.*]] = load i32, ptr [[TMP62]], align 8
+; AVX1-NEXT: [[TMP91:%.*]] = load i32, ptr [[TMP63]], align 8
+; AVX1-NEXT: [[TMP92:%.*]] = insertelement <4 x i32> poison, i32 [[TMP88]], i32 0
+; AVX1-NEXT: [[TMP93:%.*]] = insertelement <4 x i32> [[TMP92]], i32 [[TMP89]], i32 1
+; AVX1-NEXT: [[TMP94:%.*]] = insertelement <4 x i32> [[TMP93]], i32 [[TMP90]], i32 2
+; AVX1-NEXT: [[TMP95:%.*]] = insertelement <4 x i32> [[TMP94]], i32 [[TMP91]], i32 3
+; AVX1-NEXT: [[TMP96:%.*]] = icmp ne <4 x i32> [[TMP71]], zeroinitializer
+; AVX1-NEXT: [[TMP97:%.*]] = icmp ne <4 x i32> [[TMP79]], zeroinitializer
+; AVX1-NEXT: [[TMP98:%.*]] = icmp ne <4 x i32> [[TMP87]], zeroinitializer
+; AVX1-NEXT: [[TMP99:%.*]] = icmp ne <4 x i32> [[TMP95]], zeroinitializer
+; AVX1-NEXT: [[TMP100]] = or <4 x i1> [[VEC_PHI]], [[TMP96]]
+; AVX1-NEXT: [[TMP101]] = or <4 x i1> [[VEC_PHI1]], [[TMP97]]
+; AVX1-NEXT: [[TMP102]] = or <4 x i1> [[VEC_PHI2]], [[TMP98]]
+; AVX1-NEXT: [[TMP103]] = or <4 x i1> [[VEC_PHI3]], [[TMP99]]
+; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; AVX1-NEXT: [[TMP104:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; AVX1-NEXT: br i1 [[TMP104]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
; AVX1: [[EXIT]]:
-; AVX1-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ]
-; AVX1-NEXT: ret i32 [[SEL_LCSSA]]
+; AVX1-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP101]], [[TMP100]]
+; AVX1-NEXT: [[BIN_RDX4:%.*]] = or <4 x i1> [[TMP102]], [[BIN_RDX]]
+; AVX1-NEXT: [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP103]], [[BIN_RDX4]]
+; AVX1-NEXT: [[TMP105:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
+; AVX1-NEXT: [[TMP106:%.*]] = freeze i1 [[TMP105]]
+; AVX1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP106]], i32 0, i32 1
+; AVX1-NEXT: br i1 false, [[EXIT1:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX1: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29]]
+; AVX1: [[VEC_EPILOG_PH]]:
+; AVX1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX1-NEXT: [[TMP107:%.*]] = icmp ne i32 [[BC_MERGE_RDX]], 1
+; AVX1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP107]], i64 0
+; AVX1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; AVX1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX1: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX1-NEXT: [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX1-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i1> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH]] ], [ [[TMP133:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX1-NEXT: [[TMP108:%.*]] = sub i32 100, [[INDEX6]]
+; AVX1-NEXT: [[TMP109:%.*]] = add i32 [[TMP108]], -1
+; AVX1-NEXT: [[TMP110:%.*]] = add i32 [[TMP108]], -2
+; AVX1-NEXT: [[TMP111:%.*]] = add i32 [[TMP108]], -3
+; AVX1-NEXT: [[TMP112:%.*]] = zext i32 [[TMP108]] to i64
+; AVX1-NEXT: [[TMP113:%.*]] = zext i32 [[TMP109]] to i64
+; AVX1-NEXT: [[TMP114:%.*]] = zext i32 [[TMP110]] to i64
+; AVX1-NEXT: [[TMP115:%.*]] = zext i32 [[TMP111]] to i64
+; AVX1-NEXT: [[TMP116:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP112]]
+; AVX1-NEXT: [[TMP117:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP113]]
+; AVX1-NEXT: [[TMP118:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP114]]
+; AVX1-NEXT: [[TMP119:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP115]]
+; AVX1-NEXT: [[TMP120:%.*]] = load ptr, ptr [[TMP116]], align 8
+; AVX1-NEXT: [[TMP121:%.*]] = load ptr, ptr [[TMP117]], align 8
+; AVX1-NEXT: [[TMP122:%.*]] = load ptr, ptr [[TMP118]], align 8
+; AVX1-NEXT: [[TMP123:%.*]] = load ptr, ptr [[TMP119]], align 8
+; AVX1-NEXT: [[TMP124:%.*]] = load i32, ptr [[TMP120]], align 8
+; AVX1-NEXT: [[TMP125:%.*]] = load i32, ptr [[TMP121]], align 8
+; AVX1-NEXT: [[TMP126:%.*]] = load i32, ptr [[TMP122]], align 8
+; AVX1-NEXT: [[TMP127:%.*]] = load i32, ptr [[TMP123]], align 8
+; AVX1-NEXT: [[TMP128:%.*]] = insertelement <4 x i32> poison, i32 [[TMP124]], i32 0
+; AVX1-NEXT: [[TMP129:%.*]] = insertelement <4 x i32> [[TMP128]], i32 [[TMP125]], i32 1
+; AVX1-NEXT: [[TMP130:%.*]] = insertelement <4 x i32> [[TMP129]], i32 [[TMP126]], i32 2
+; AVX1-NEXT: [[TMP131:%.*]] = insertelement <4 x i32> [[TMP130]], i32 [[TMP127]], i32 3
+; AVX1-NEXT: [[TMP132:%.*]] = icmp ne <4 x i32> [[TMP131]], zeroinitializer
+; AVX1-NEXT: [[TMP133]] = or <4 x i1> [[VEC_PHI7]], [[TMP132]]
+; AVX1-NEXT: [[INDEX_NEXT8]] = add nuw i32 [[INDEX6]], 4
+; AVX1-NEXT: [[TMP134:%.*]] = icmp eq i32 [[INDEX_NEXT8]], 100
+; AVX1-NEXT: br i1 [[TMP134]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; AVX1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX1-NEXT: [[TMP135:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP133]])
+; AVX1-NEXT: [[TMP136:%.*]] = freeze i1 [[TMP135]]
+; AVX1-NEXT: [[RDX_SELECT9:%.*]] = select i1 [[TMP136]], i32 0, i32 1
+; AVX1-NEXT: br i1 true, [[EXIT1]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX1: [[VEC_EPILOG_SCALAR_PH]]:
;
; AVX2-LABEL: define i32 @reverse_gather(
; AVX2-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
index 28ef33f2cef3b..e31df6c748c75 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
@@ -93,24 +93,108 @@ exit:
define i32 @replicating_loads_and_reduction(ptr %A, ptr %B) #0 {
; CHECK-LABEL: define i32 @replicating_loads_and_reduction(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[LOOP]] ], [ [[TMP67:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[LOOP]] ], [ [[TMP68:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[IV]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[IV]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[IV]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[IV]], 5
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[IV]], 6
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[IV]], 7
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[IV]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP0]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP4]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[TMP6]]
; CHECK-NEXT: [[L_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP8]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP12]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[TMP13]], align 1
+; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP14]], align 1
; CHECK-NEXT: [[A_EXT:%.*]] = sext i8 [[L_A]] to i32
+; CHECK-NEXT: [[TMP24:%.*]] = sext i8 [[TMP16]] to i32
+; CHECK-NEXT: [[TMP25:%.*]] = sext i8 [[TMP17]] to i32
+; CHECK-NEXT: [[TMP26:%.*]] = sext i8 [[TMP18]] to i32
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[A_EXT]], i32 0
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i32 1
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP25]], i32 2
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP26]], i32 3
+; CHECK-NEXT: [[TMP31:%.*]] = sext i8 [[TMP19]] to i32
+; CHECK-NEXT: [[TMP32:%.*]] = sext i8 [[TMP20]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = sext i8 [[TMP21]] to i32
+; CHECK-NEXT: [[TMP34:%.*]] = sext i8 [[TMP22]] to i32
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> poison, i32 [[TMP31]], i32 0
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP32]], i32 1
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP33]], i32 2
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP34]], i32 3
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[A_EXT]]
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP24]]
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP25]]
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP26]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP31]]
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP32]]
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP33]]
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP34]]
; CHECK-NEXT: [[L_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[B_EXT:%.*]] = sext i8 [[L_B]] to i32
-; CHECK-NEXT: [[ADD:%.*]] = add i32 [[RED]], [[A_EXT]]
+; CHECK-NEXT: [[TMP48:%.*]] = load i8, ptr [[TMP40]], align 1
+; CHECK-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP41]], align 1
+; CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[TMP42]], align 1
+; CHECK-NEXT: [[TMP51:%.*]] = insertelement <4 x i8> poison, i8 [[L_B]], i32 0
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i8> [[TMP51]], i8 [[TMP48]], i32 1
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <4 x i8> [[TMP52]], i8 [[TMP49]], i32 2
+; CHECK-NEXT: [[TMP54:%.*]] = insertelement <4 x i8> [[TMP53]], i8 [[TMP50]], i32 3
+; CHECK-NEXT: [[TMP55:%.*]] = load i8, ptr [[TMP43]], align 1
+; CHECK-NEXT: [[TMP56:%.*]] = load i8, ptr [[TMP44]], align 1
+; CHECK-NEXT: [[TMP57:%.*]] = load i8, ptr [[TMP45]], align 1
+; CHECK-NEXT: [[TMP58:%.*]] = load i8, ptr [[TMP46]], align 1
+; CHECK-NEXT: [[TMP59:%.*]] = insertelement <4 x i8> poison, i8 [[TMP55]], i32 0
+; CHECK-NEXT: [[TMP60:%.*]] = insertelement <4 x i8> [[TMP59]], i8 [[TMP56]], i32 1
+; CHECK-NEXT: [[TMP61:%.*]] = insertelement <4 x i8> [[TMP60]], i8 [[TMP57]], i32 2
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <4 x i8> [[TMP61]], i8 [[TMP58]], i32 3
+; CHECK-NEXT: [[TMP63:%.*]] = sext <4 x i8> [[TMP54]] to <4 x i32>
+; CHECK-NEXT: [[TMP64:%.*]] = sext <4 x i8> [[TMP62]] to <4 x i32>
+; CHECK-NEXT: [[TMP65:%.*]] = add <4 x i32> [[VEC_PHI]], [[TMP30]]
+; CHECK-NEXT: [[TMP66:%.*]] = add <4 x i32> [[VEC_PHI1]], [[TMP38]]
+; CHECK-NEXT: [[TMP67]] = add <4 x i32> [[TMP65]], [[TMP63]]
+; CHECK-NEXT: [[TMP68]] = add <4 x i32> [[TMP66]], [[TMP64]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[IV]], 8
+; CHECK-NEXT: [[TMP69:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP69]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP68]], [[TMP67]]
+; CHECK-NEXT: [[TMP70:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[TMP70]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i32 [[IV1]]
+; CHECK-NEXT: [[L_A1:%.*]] = load i8, ptr [[GEP_A1]], align 1
+; CHECK-NEXT: [[A_EXT1:%.*]] = sext i8 [[L_A1]] to i32
+; CHECK-NEXT: [[GEP_B1:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[A_EXT1]]
+; CHECK-NEXT: [[L_B1:%.*]] = load i8, ptr [[GEP_B1]], align 1
+; CHECK-NEXT: [[B_EXT:%.*]] = sext i8 [[L_B1]] to i32
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[RED]], [[A_EXT1]]
; CHECK-NEXT: [[RED_NEXT]] = add i32 [[ADD]], [[B_EXT]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP1]] ]
; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs-max-bandwidth.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs-max-bandwidth.ll
index a33874a871443..1f83de7078474 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs-max-bandwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs-max-bandwidth.ll
@@ -292,14 +292,274 @@ exit:
define void @replicating_store_with_phi_addr2(ptr noalias %array, ptr noalias %base, i64 %N, i32 %x, i1 %cond) {
; CHECK-LABEL: define void @replicating_store_with_phi_addr2(
; CHECK-SAME: ptr noalias [[ARRAY:%.*]], ptr noalias [[BASE:%.*]], i64 [[N:%.*]], i32 [[X:%.*]], i1 [[COND:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[PTR1:%.*]] = alloca i8, align 1
; CHECK-NEXT: [[PTR2:%.*]] = alloca i8, align 1
; CHECK-NEXT: [[PTR3:%.*]] = alloca i8, align 1
; CHECK-NEXT: call void @init(ptr [[PTR1]], ptr [[PTR2]], ptr [[PTR3]])
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x ptr> poison, ptr [[PTR1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x ptr> [[BROADCAST_SPLATINSERT]], <16 x ptr> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT1]], <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE32:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP13]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[ARRAY]], i64 [[TMP14]]
+; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = load i32, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP25]], align 8
+; CHECK-NEXT: [[TMP42:%.*]] = load i32, ptr [[TMP26]], align 8
+; CHECK-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP27]], align 8
+; CHECK-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP28]], align 8
+; CHECK-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP29]], align 8
+; CHECK-NEXT: [[TMP46:%.*]] = load i32, ptr [[TMP30]], align 8
+; CHECK-NEXT: [[TMP47:%.*]] = load ptr, ptr [[BASE]], align 4
+; CHECK-NEXT: [[TMP48:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP49:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP51:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP54:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP56:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP57:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP58:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP59:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP60:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP61:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP62:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP63:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT: [[TMP64:%.*]] = insertelement <16 x i32> poison, i32 [[TMP48]], i32 0
+; CHECK-NEXT: [[TMP65:%.*]] = insertelement <16 x i32> [[TMP64]], i32 [[TMP49]], i32 1
+; CHECK-NEXT: [[TMP66:%.*]] = insertelement <16 x i32> [[TMP65]], i32 [[TMP50]], i32 2
+; CHECK-NEXT: [[TMP67:%.*]] = insertelement <16 x i32> [[TMP66]], i32 [[TMP51]], i32 3
+; CHECK-NEXT: [[TMP68:%.*]] = insertelement <16 x i32> [[TMP67]], i32 [[TMP52]], i32 4
+; CHECK-NEXT: [[TMP69:%.*]] = insertelement <16 x i32> [[TMP68]], i32 [[TMP53]], i32 5
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <16 x i32> [[TMP69]], i32 [[TMP54]], i32 6
+; CHECK-NEXT: [[TMP71:%.*]] = insertelement <16 x i32> [[TMP70]], i32 [[TMP55]], i32 7
+; CHECK-NEXT: [[TMP72:%.*]] = insertelement <16 x i32> [[TMP71]], i32 [[TMP56]], i32 8
+; CHECK-NEXT: [[TMP73:%.*]] = insertelement <16 x i32> [[TMP72]], i32 [[TMP57]], i32 9
+; CHECK-NEXT: [[TMP74:%.*]] = insertelement <16 x i32> [[TMP73]], i32 [[TMP58]], i32 10
+; CHECK-NEXT: [[TMP75:%.*]] = insertelement <16 x i32> [[TMP74]], i32 [[TMP59]], i32 11
+; CHECK-NEXT: [[TMP76:%.*]] = insertelement <16 x i32> [[TMP75]], i32 [[TMP60]], i32 12
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <16 x i32> [[TMP76]], i32 [[TMP61]], i32 13
+; CHECK-NEXT: [[TMP78:%.*]] = insertelement <16 x i32> [[TMP77]], i32 [[TMP62]], i32 14
+; CHECK-NEXT: [[TMP79:%.*]] = insertelement <16 x i32> [[TMP78]], i32 [[TMP63]], i32 15
+; CHECK-NEXT: [[TMP80:%.*]] = icmp sgt <16 x i32> [[TMP79]], zeroinitializer
+; CHECK-NEXT: [[TMP81:%.*]] = xor <16 x i1> [[TMP80]], splat (i1 true)
+; CHECK-NEXT: [[TMP82:%.*]] = icmp slt i32 [[X]], [[TMP31]]
+; CHECK-NEXT: [[TMP83:%.*]] = icmp slt i32 [[X]], [[TMP32]]
+; CHECK-NEXT: [[TMP84:%.*]] = icmp slt i32 [[X]], [[TMP33]]
+; CHECK-NEXT: [[TMP85:%.*]] = icmp slt i32 [[X]], [[TMP34]]
+; CHECK-NEXT: [[TMP86:%.*]] = icmp slt i32 [[X]], [[TMP35]]
+; CHECK-NEXT: [[TMP87:%.*]] = icmp slt i32 [[X]], [[TMP36]]
+; CHECK-NEXT: [[TMP88:%.*]] = icmp slt i32 [[X]], [[TMP37]]
+; CHECK-NEXT: [[TMP89:%.*]] = icmp slt i32 [[X]], [[TMP38]]
+; CHECK-NEXT: [[TMP90:%.*]] = icmp slt i32 [[X]], [[TMP39]]
+; CHECK-NEXT: [[TMP91:%.*]] = icmp slt i32 [[X]], [[TMP40]]
+; CHECK-NEXT: [[TMP92:%.*]] = icmp slt i32 [[X]], [[TMP41]]
+; CHECK-NEXT: [[TMP93:%.*]] = icmp slt i32 [[X]], [[TMP42]]
+; CHECK-NEXT: [[TMP94:%.*]] = icmp slt i32 [[X]], [[TMP43]]
+; CHECK-NEXT: [[TMP95:%.*]] = icmp slt i32 [[X]], [[TMP44]]
+; CHECK-NEXT: [[TMP96:%.*]] = icmp slt i32 [[X]], [[TMP45]]
+; CHECK-NEXT: [[TMP97:%.*]] = icmp slt i32 [[X]], [[TMP46]]
+; CHECK-NEXT: [[TMP98:%.*]] = select i1 [[TMP82]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP99:%.*]] = select i1 [[TMP83]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP100:%.*]] = select i1 [[TMP84]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP101:%.*]] = select i1 [[TMP85]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP102:%.*]] = select i1 [[TMP86]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP103:%.*]] = select i1 [[TMP87]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP104:%.*]] = select i1 [[TMP88]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP105:%.*]] = select i1 [[TMP89]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP106:%.*]] = select i1 [[TMP90]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP107:%.*]] = select i1 [[TMP91]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP108:%.*]] = select i1 [[TMP92]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP109:%.*]] = select i1 [[TMP93]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP110:%.*]] = select i1 [[TMP94]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP111:%.*]] = select i1 [[TMP95]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP112:%.*]] = select i1 [[TMP96]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP113:%.*]] = select i1 [[TMP97]], ptr [[PTR2]], ptr [[PTR3]]
+; CHECK-NEXT: [[TMP114:%.*]] = insertelement <16 x ptr> poison, ptr [[TMP98]], i32 0
+; CHECK-NEXT: [[TMP115:%.*]] = insertelement <16 x ptr> [[TMP114]], ptr [[TMP99]], i32 1
+; CHECK-NEXT: [[TMP116:%.*]] = insertelement <16 x ptr> [[TMP115]], ptr [[TMP100]], i32 2
+; CHECK-NEXT: [[TMP117:%.*]] = insertelement <16 x ptr> [[TMP116]], ptr [[TMP101]], i32 3
+; CHECK-NEXT: [[TMP118:%.*]] = insertelement <16 x ptr> [[TMP117]], ptr [[TMP102]], i32 4
+; CHECK-NEXT: [[TMP119:%.*]] = insertelement <16 x ptr> [[TMP118]], ptr [[TMP103]], i32 5
+; CHECK-NEXT: [[TMP120:%.*]] = insertelement <16 x ptr> [[TMP119]], ptr [[TMP104]], i32 6
+; CHECK-NEXT: [[TMP121:%.*]] = insertelement <16 x ptr> [[TMP120]], ptr [[TMP105]], i32 7
+; CHECK-NEXT: [[TMP122:%.*]] = insertelement <16 x ptr> [[TMP121]], ptr [[TMP106]], i32 8
+; CHECK-NEXT: [[TMP123:%.*]] = insertelement <16 x ptr> [[TMP122]], ptr [[TMP107]], i32 9
+; CHECK-NEXT: [[TMP124:%.*]] = insertelement <16 x ptr> [[TMP123]], ptr [[TMP108]], i32 10
+; CHECK-NEXT: [[TMP125:%.*]] = insertelement <16 x ptr> [[TMP124]], ptr [[TMP109]], i32 11
+; CHECK-NEXT: [[TMP126:%.*]] = insertelement <16 x ptr> [[TMP125]], ptr [[TMP110]], i32 12
+; CHECK-NEXT: [[TMP127:%.*]] = insertelement <16 x ptr> [[TMP126]], ptr [[TMP111]], i32 13
+; CHECK-NEXT: [[TMP128:%.*]] = insertelement <16 x ptr> [[TMP127]], ptr [[TMP112]], i32 14
+; CHECK-NEXT: [[TMP129:%.*]] = insertelement <16 x ptr> [[TMP128]], ptr [[TMP113]], i32 15
+; CHECK-NEXT: [[TMP130:%.*]] = select <16 x i1> [[TMP80]], <16 x i1> [[BROADCAST_SPLAT2]], <16 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP131:%.*]] = or <16 x i1> [[TMP130]], [[TMP81]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <16 x i1> [[TMP80]], <16 x ptr> [[BROADCAST_SPLAT]], <16 x ptr> [[TMP129]]
+; CHECK-NEXT: [[TMP132:%.*]] = extractelement <16 x i1> [[TMP131]], i64 0
+; CHECK-NEXT: br i1 [[TMP132]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 0
+; CHECK-NEXT: store i8 0, ptr [[TMP133]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP134:%.*]] = extractelement <16 x i1> [[TMP131]], i64 1
+; CHECK-NEXT: br i1 [[TMP134]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP135:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 1
+; CHECK-NEXT: store i8 0, ptr [[TMP135]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP136:%.*]] = extractelement <16 x i1> [[TMP131]], i64 2
+; CHECK-NEXT: br i1 [[TMP136]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP137:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 2
+; CHECK-NEXT: store i8 0, ptr [[TMP137]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP138:%.*]] = extractelement <16 x i1> [[TMP131]], i64 3
+; CHECK-NEXT: br i1 [[TMP138]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP139:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 3
+; CHECK-NEXT: store i8 0, ptr [[TMP139]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[TMP140:%.*]] = extractelement <16 x i1> [[TMP131]], i64 4
+; CHECK-NEXT: br i1 [[TMP140]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK: [[PRED_STORE_IF9]]:
+; CHECK-NEXT: [[TMP141:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 4
+; CHECK-NEXT: store i8 0, ptr [[TMP141]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; CHECK: [[PRED_STORE_CONTINUE10]]:
+; CHECK-NEXT: [[TMP142:%.*]] = extractelement <16 x i1> [[TMP131]], i64 5
+; CHECK-NEXT: br i1 [[TMP142]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; CHECK: [[PRED_STORE_IF11]]:
+; CHECK-NEXT: [[TMP143:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 5
+; CHECK-NEXT: store i8 0, ptr [[TMP143]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; CHECK: [[PRED_STORE_CONTINUE12]]:
+; CHECK-NEXT: [[TMP144:%.*]] = extractelement <16 x i1> [[TMP131]], i64 6
+; CHECK-NEXT: br i1 [[TMP144]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; CHECK: [[PRED_STORE_IF13]]:
+; CHECK-NEXT: [[TMP145:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 6
+; CHECK-NEXT: store i8 0, ptr [[TMP145]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: [[TMP146:%.*]] = extractelement <16 x i1> [[TMP131]], i64 7
+; CHECK-NEXT: br i1 [[TMP146]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; CHECK: [[PRED_STORE_IF15]]:
+; CHECK-NEXT: [[TMP147:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 7
+; CHECK-NEXT: store i8 0, ptr [[TMP147]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; CHECK: [[PRED_STORE_CONTINUE16]]:
+; CHECK-NEXT: [[TMP148:%.*]] = extractelement <16 x i1> [[TMP131]], i64 8
+; CHECK-NEXT: br i1 [[TMP148]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; CHECK: [[PRED_STORE_IF17]]:
+; CHECK-NEXT: [[TMP149:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 8
+; CHECK-NEXT: store i8 0, ptr [[TMP149]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
+; CHECK: [[PRED_STORE_CONTINUE18]]:
+; CHECK-NEXT: [[TMP150:%.*]] = extractelement <16 x i1> [[TMP131]], i64 9
+; CHECK-NEXT: br i1 [[TMP150]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; CHECK: [[PRED_STORE_IF19]]:
+; CHECK-NEXT: [[TMP151:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 9
+; CHECK-NEXT: store i8 0, ptr [[TMP151]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; CHECK: [[PRED_STORE_CONTINUE20]]:
+; CHECK-NEXT: [[TMP152:%.*]] = extractelement <16 x i1> [[TMP131]], i64 10
+; CHECK-NEXT: br i1 [[TMP152]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; CHECK: [[PRED_STORE_IF21]]:
+; CHECK-NEXT: [[TMP153:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 10
+; CHECK-NEXT: store i8 0, ptr [[TMP153]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; CHECK: [[PRED_STORE_CONTINUE22]]:
+; CHECK-NEXT: [[TMP154:%.*]] = extractelement <16 x i1> [[TMP131]], i64 11
+; CHECK-NEXT: br i1 [[TMP154]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; CHECK: [[PRED_STORE_IF23]]:
+; CHECK-NEXT: [[TMP155:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 11
+; CHECK-NEXT: store i8 0, ptr [[TMP155]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; CHECK: [[PRED_STORE_CONTINUE24]]:
+; CHECK-NEXT: [[TMP156:%.*]] = extractelement <16 x i1> [[TMP131]], i64 12
+; CHECK-NEXT: br i1 [[TMP156]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; CHECK: [[PRED_STORE_IF25]]:
+; CHECK-NEXT: [[TMP157:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 12
+; CHECK-NEXT: store i8 0, ptr [[TMP157]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; CHECK: [[PRED_STORE_CONTINUE26]]:
+; CHECK-NEXT: [[TMP158:%.*]] = extractelement <16 x i1> [[TMP131]], i64 13
+; CHECK-NEXT: br i1 [[TMP158]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; CHECK: [[PRED_STORE_IF27]]:
+; CHECK-NEXT: [[TMP159:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 13
+; CHECK-NEXT: store i8 0, ptr [[TMP159]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; CHECK: [[PRED_STORE_CONTINUE28]]:
+; CHECK-NEXT: [[TMP160:%.*]] = extractelement <16 x i1> [[TMP131]], i64 14
+; CHECK-NEXT: br i1 [[TMP160]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; CHECK: [[PRED_STORE_IF29]]:
+; CHECK-NEXT: [[TMP161:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 14
+; CHECK-NEXT: store i8 0, ptr [[TMP161]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; CHECK: [[PRED_STORE_CONTINUE30]]:
+; CHECK-NEXT: [[TMP162:%.*]] = extractelement <16 x i1> [[TMP131]], i64 15
+; CHECK-NEXT: br i1 [[TMP162]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_IF31]]:
+; CHECK-NEXT: [[TMP163:%.*]] = extractelement <16 x ptr> [[PREDPHI]], i64 15
+; CHECK-NEXT: store i8 0, ptr [[TMP163]], align 1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_CONTINUE32]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP164:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP164]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV]], 99
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_BODY:.*]], label %[[EXIT:.*]]
; CHECK: [[LOOP_BODY]]:
@@ -321,7 +581,7 @@ define void @replicating_store_with_phi_addr2(ptr noalias %array, ptr noalias %b
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: br label %[[LOOP_HEADER]]
+; CHECK-NEXT: br label %[[LOOP_HEADER1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -375,4 +635,6 @@ attributes #0 = { "target-cpu"="znver2" }
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index ddb1905f3ffc4..a75e9ad550bf1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -473,7 +473,7 @@ define double @test_load_used_by_other_load_scev(ptr %ptr.a, ptr %ptr.b, ptr %pt
; I64-NEXT: br label %[[VECTOR_BODY1:.*]]
; I64: [[VECTOR_BODY1]]:
; I64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
-; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I64-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; I64-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
More information about the llvm-commits
mailing list