[llvm] [LV] Factor costInterleaveGatherScatter (NFC) (PR #215857)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 23:40:16 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/215857
>From dd4ed817c9263cf7a4d4290bb35b4bbca5425cce Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 12 Aug 2026 18:36:44 +0100
Subject: [PATCH 1/3] [LV] Use ScalarTy in isLegalGatherOrScatter
The rationale for using the scalar type in isLegalMaskedLoadOrStore
should apply: the use-sites do not clamp the VF range based on this
legality check anyway.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 2 -
.../AArch64/aggressive-interleaving.ll | 147 ++++++---
.../LoopVectorize/AArch64/blend-costs.ll | 27 +-
.../AArch64/conditional-branches-cost.ll | 179 +++++++----
.../AArch64/drop-poison-generating-flags.ll | 37 +--
.../first-order-recurrence-fold-tail.ll | 49 +--
.../LoopVectorize/AArch64/induction-costs.ll | 66 ++---
.../AArch64/interleave-with-gaps.ll | 50 +---
.../AArch64/partial-reduce-dot-product.ll | 279 +-----------------
.../LoopVectorize/AArch64/predicated-costs.ll | 137 ++-------
.../AArch64/reduction-recurrence-costs-sve.ll | 278 +++++------------
.../AArch64/sve2-histcnt-epilogue.ll | 7 +-
...row-interleave-to-widen-memory-scalable.ll | 39 +--
.../widen-gep-all-indices-invariant.ll | 24 +-
14 files changed, 421 insertions(+), 900 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index ff9b9171d8c8c..dc4b640eff109 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -154,8 +154,6 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
return false;
auto *Ty = getLoadStoreType(V);
Align Align = getLoadStoreAlignment(V);
- if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
return ForceTargetSupportsGatherScatterOps ||
(LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
index 8a6de3f6a7ba6..205d2e7492bd6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
@@ -17,21 +17,24 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320: [[OUTER]]:
; A320-NEXT: [[TPM26:%.*]] = add i64 0, 1
; A320-NEXT: [[TPM10:%.*]] = alloca i32, align 8
+; A320-NEXT: [[TPM102:%.*]] = ptrtoaddr ptr [[TPM10]] to i64
; A320-NEXT: [[TPM27:%.*]] = getelementptr inbounds i32, ptr [[TPM10]], i64 [[TPM26]]
; A320-NEXT: [[TPM28:%.*]] = getelementptr inbounds ptr, ptr [[TPM15]], i64 0
; A320-NEXT: [[TPM29:%.*]] = load ptr, ptr [[TPM28]], align 8
+; A320-NEXT: [[TMP0:%.*]] = ptrtoaddr ptr [[TPM29]] to i64
; A320-NEXT: [[TPM17:%.*]] = alloca double, align 8
; A320-NEXT: [[TPM32:%.*]] = getelementptr inbounds double, ptr [[TPM17]], i64 [[TPM26]]
-; A320-NEXT: [[TMP0:%.*]] = ptrtoaddr ptr [[TPM29]] to i64
-; A320-NEXT: [[TPM102:%.*]] = ptrtoaddr ptr [[TPM10]] to i64
; A320-NEXT: [[TMP8:%.*]] = add i64 [[TMP0]], -8
; A320-NEXT: [[TMP6:%.*]] = sub i64 [[TMP8]], [[TPM102]]
; A320-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP6]], 2
; A320-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
-; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 1
+; A320-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP3]], 16
+; A320-NEXT: br i1 [[MIN_ITERS_CHECK3]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; A320: [[VECTOR_PH1]]:
+; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 15
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; A320-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
; A320-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP4]]
@@ -39,47 +42,103 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[IND_END3:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP5]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
-; A320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; A320-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
-; A320-NEXT: [[VEC_PHI5:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI4:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI5:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI6:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
; A320-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
-; A320-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 4
; A320-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[OFFSET_IDX]]
+; A320-NEXT: [[TMP10:%.*]] = shl i64 [[INDEX]], 3
+; A320-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP10]]
+; A320-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 4
+; A320-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 8
+; A320-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[NEXT_GEP7]], i64 12
+; A320-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[NEXT_GEP7]], align 8
+; A320-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP11]], align 8
+; A320-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP12]], align 8
+; A320-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
+; A320-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 4
+; A320-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 8
+; A320-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 12
+; A320-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
+; A320-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4
+; A320-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4
+; A320-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
+; A320-NEXT: [[TMP17:%.*]] = zext <4 x i32> [[WIDE_LOAD11]] to <4 x i64>
+; A320-NEXT: [[TMP18:%.*]] = zext <4 x i32> [[WIDE_LOAD12]] to <4 x i64>
+; A320-NEXT: [[TMP19:%.*]] = zext <4 x i32> [[WIDE_LOAD13]] to <4 x i64>
+; A320-NEXT: [[TMP20:%.*]] = zext <4 x i32> [[WIDE_LOAD14]] to <4 x i64>
+; A320-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds double, ptr [[TPM19]], <4 x i64> [[TMP17]]
+; A320-NEXT: [[WIDE_GEP15:%.*]] = getelementptr inbounds double, ptr [[TPM19]], <4 x i64> [[TMP18]]
+; A320-NEXT: [[WIDE_GEP16:%.*]] = getelementptr inbounds double, ptr [[TPM19]], <4 x i64> [[TMP19]]
+; A320-NEXT: [[WIDE_GEP17:%.*]] = getelementptr inbounds double, ptr [[TPM19]], <4 x i64> [[TMP20]]
+; A320-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x double> poison)
+; A320-NEXT: [[WIDE_MASKED_GATHER18:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP15]], <4 x i1> splat (i1 true), <4 x double> poison)
+; A320-NEXT: [[WIDE_MASKED_GATHER19:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP16]], <4 x i1> splat (i1 true), <4 x double> poison)
+; A320-NEXT: [[WIDE_MASKED_GATHER20:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP17]], <4 x i1> splat (i1 true), <4 x double> poison)
+; A320-NEXT: [[TMP21:%.*]] = fmul fast <4 x double> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]
+; A320-NEXT: [[TMP22:%.*]] = fmul fast <4 x double> [[WIDE_MASKED_GATHER18]], [[WIDE_LOAD8]]
+; A320-NEXT: [[TMP23:%.*]] = fmul fast <4 x double> [[WIDE_MASKED_GATHER19]], [[WIDE_LOAD9]]
+; A320-NEXT: [[TMP24:%.*]] = fmul fast <4 x double> [[WIDE_MASKED_GATHER20]], [[WIDE_LOAD10]]
+; A320-NEXT: [[TMP25]] = fadd fast <4 x double> [[TMP21]], [[VEC_PHI]]
+; A320-NEXT: [[TMP26]] = fadd fast <4 x double> [[TMP22]], [[VEC_PHI4]]
+; A320-NEXT: [[TMP27]] = fadd fast <4 x double> [[TMP23]], [[VEC_PHI5]]
+; A320-NEXT: [[TMP28]] = fadd fast <4 x double> [[TMP24]], [[VEC_PHI6]]
+; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; A320-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; A320-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; A320: [[MIDDLE_BLOCK1]]:
+; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x double> [[TMP26]], [[TMP25]]
+; A320-NEXT: [[BIN_RDX21:%.*]] = fadd fast <4 x double> [[TMP27]], [[BIN_RDX]]
+; A320-NEXT: [[BIN_RDX22:%.*]] = fadd fast <4 x double> [[TMP28]], [[BIN_RDX21]]
+; A320-NEXT: [[TMP30:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[BIN_RDX22]])
+; A320-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; A320-NEXT: br i1 [[CMP_N1]], label %[[EXIT_INNER:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; A320: [[VEC_EPILOG_ITER_CHECK]]:
+; A320-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; A320-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; A320: [[VEC_EPILOG_PH]]:
+; A320-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP30]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_PH]] ]
+; A320-NEXT: [[TMP31:%.*]] = and i64 [[TMP3]], 3
+; A320-NEXT: [[N_VEC24:%.*]] = sub i64 [[TMP3]], [[TMP31]]
+; A320-NEXT: [[TMP7:%.*]] = shl i64 [[N_VEC24]], 2
; A320-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP7]]
-; A320-NEXT: [[OFFSET_IDX7:%.*]] = shl i64 [[INDEX]], 3
-; A320-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX7]], 8
+; A320-NEXT: [[OFFSET_IDX7:%.*]] = shl i64 [[N_VEC24]], 3
; A320-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[OFFSET_IDX7]]
+; A320-NEXT: [[TMP36:%.*]] = insertelement <4 x double> zeroinitializer, double [[BC_MERGE_RDX]], i64 0
+; A320-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; A320: [[VEC_EPILOG_VECTOR_BODY]]:
+; A320-NEXT: [[INDEX25:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; A320-NEXT: [[VEC_PHI26:%.*]] = phi <4 x double> [ [[TMP36]], %[[VEC_EPILOG_PH]] ], [ [[TMP41:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; A320-NEXT: [[TMP37:%.*]] = shl i64 [[INDEX25]], 2
+; A320-NEXT: [[NEXT_GEP27:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP37]]
+; A320-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX25]], 3
; A320-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[TPM32]], i64 [[TMP9]]
-; A320-NEXT: [[TMP10:%.*]] = load double, ptr [[NEXT_GEP8]], align 8
-; A320-NEXT: [[TMP11:%.*]] = load double, ptr [[NEXT_GEP9]], align 8
-; A320-NEXT: [[TMP12:%.*]] = load i32, ptr [[NEXT_GEP]], align 4
-; A320-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP6]], align 4
-; A320-NEXT: [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
-; A320-NEXT: [[TMP15:%.*]] = zext i32 [[TMP13]] to i64
-; A320-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP14]]
-; A320-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 [[TMP15]]
-; A320-NEXT: [[TMP18:%.*]] = load double, ptr [[TMP16]], align 8
-; A320-NEXT: [[TMP19:%.*]] = load double, ptr [[TMP17]], align 8
-; A320-NEXT: [[TMP20:%.*]] = fmul fast double [[TMP18]], [[TMP10]]
-; A320-NEXT: [[TMP21:%.*]] = fmul fast double [[TMP19]], [[TMP11]]
-; A320-NEXT: [[TMP22]] = fadd fast double [[TMP20]], [[VEC_PHI]]
-; A320-NEXT: [[TMP23]] = fadd fast double [[TMP21]], [[VEC_PHI5]]
-; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; A320-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; A320-NEXT: [[WIDE_LOAD29:%.*]] = load <4 x double>, ptr [[NEXT_GEP9]], align 8
+; A320-NEXT: [[WIDE_LOAD30:%.*]] = load <4 x i32>, ptr [[NEXT_GEP27]], align 4
+; A320-NEXT: [[TMP39:%.*]] = zext <4 x i32> [[WIDE_LOAD30]] to <4 x i64>
+; A320-NEXT: [[WIDE_GEP31:%.*]] = getelementptr inbounds double, ptr [[TPM19]], <4 x i64> [[TMP39]]
+; A320-NEXT: [[WIDE_MASKED_GATHER32:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP31]], <4 x i1> splat (i1 true), <4 x double> poison)
+; A320-NEXT: [[TMP40:%.*]] = fmul fast <4 x double> [[WIDE_MASKED_GATHER32]], [[WIDE_LOAD29]]
+; A320-NEXT: [[TMP41]] = fadd fast <4 x double> [[TMP40]], [[VEC_PHI26]]
+; A320-NEXT: [[INDEX_NEXT33]] = add nuw i64 [[INDEX25]], 4
+; A320-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT33]], [[N_VEC24]]
+; A320-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
-; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast double [[TMP23]], [[TMP22]]
-; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER:.*]], label %[[SCALAR_PH]]
+; A320-NEXT: [[TMP43:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP41]])
+; A320-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC24]]
+; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER]], label %[[SCALAR_PH]]
; A320: [[SCALAR_PH]]:
-; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TPM27]], %[[OUTER]] ]
-; A320-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END3]], %[[MIDDLE_BLOCK]] ], [ [[TPM32]], %[[OUTER]] ]
-; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[OUTER]] ]
+; A320-NEXT: [[BC_RESUME_VAL35:%.*]] = phi ptr [ [[NEXT_GEP6]], %[[MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TPM27]], %[[OUTER]] ]
+; A320-NEXT: [[BC_RESUME_VAL36:%.*]] = phi ptr [ [[NEXT_GEP8]], %[[MIDDLE_BLOCK]] ], [ [[IND_END3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[TPM32]], %[[OUTER]] ]
+; A320-NEXT: [[BC_MERGE_RDX37:%.*]] = phi double [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ [[TMP30]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[OUTER]] ]
; A320-NEXT: br label %[[INNER:.*]]
; A320: [[INNER]]:
-; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
-; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL35]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL36]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX37]], %[[SCALAR_PH]] ]
; A320-NEXT: [[TPM44:%.*]] = load double, ptr [[PHI_PTR_F64]], align 8
; A320-NEXT: [[TPM45:%.*]] = load i32, ptr [[PHI_PTR_I32]], align 4
; A320-NEXT: [[TPM46:%.*]] = zext i32 [[TPM45]] to i64
@@ -90,9 +149,9 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[NEXT_I32]] = getelementptr inbounds i32, ptr [[PHI_PTR_I32]], i64 1
; A320-NEXT: [[NEXT_F64]] = getelementptr inbounds double, ptr [[PHI_PTR_F64]], i64 1
; A320-NEXT: [[DONE:%.*]] = icmp eq ptr [[NEXT_I32]], [[TPM29]]
-; A320-NEXT: br i1 [[DONE]], label %[[EXIT_INNER]], label %[[INNER]], !llvm.loop [[LOOP3:![0-9]+]]
+; A320-NEXT: br i1 [[DONE]], label %[[EXIT_INNER]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]]
; A320: [[EXIT_INNER]]:
-; A320-NEXT: [[TPM50_LCSSA:%.*]] = phi double [ [[TPM50]], %[[INNER]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
+; A320-NEXT: [[TPM50_LCSSA:%.*]] = phi double [ [[TPM50]], %[[INNER]] ], [ [[TMP30]], %[[MIDDLE_BLOCK1]] ], [ [[TMP43]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: [[TPM35:%.*]] = getelementptr inbounds double, ptr [[TPM19]], i64 0
; A320-NEXT: [[TPM37:%.*]] = fsub fast double 0.000000e+00, [[TPM50_LCSSA]]
; A320-NEXT: store double [[TPM37]], ptr [[TPM35]], align 8
@@ -177,7 +236,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[TMP7]] = fadd fast <2 x double> [[VEC_PHI3]], [[WIDE_LOAD6]]
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; A320-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP2]]
; A320-NEXT: [[BIN_RDX7:%.*]] = fadd fast <2 x double> [[TMP10]], [[BIN_RDX]]
@@ -197,7 +256,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[SUM_NEXT]] = fadd fast double [[SUM]], [[VAL]]
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
; A320-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi double [ [[SUM_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
@@ -280,7 +339,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[TMP15]] = fadd fast <2 x double> [[VEC_PHI3]], [[TMP11]]
; A320-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 8
; A320-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; A320-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; A320: [[MIDDLE_BLOCK]]:
; A320-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP6]]
; A320-NEXT: [[BIN_RDX11:%.*]] = fadd fast <2 x double> [[TMP16]], [[BIN_RDX]]
@@ -303,7 +362,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[ACC_NEXT]] = fadd fast double [[ACC]], [[PROD]]
; A320-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; A320-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
-; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; A320-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; A320: [[EXIT_LOOPEXIT]]:
; A320-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi double [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP17]], %[[MIDDLE_BLOCK]] ]
; A320-NEXT: br label %[[EXIT]]
@@ -342,9 +401,11 @@ exit:
; A320: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; A320: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; A320: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; A320: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; A320: [[PROF3]] = !{!"branch_weights", i32 4, i32 12}
; A320: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; A320: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
; A320: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; A320: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; A320: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; A320: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index c41e9d78de209..6ca0861cd875d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -492,30 +492,15 @@ define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr n
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[Q]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], -1
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT]])
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[P1]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 5822b91d54226..01e3413020aa7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -352,7 +352,10 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: [[ENTRY:.*:]]
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 28
+; DEFAULT-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
+; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP9]], i64 32)
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP10]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[E]], i64 4
@@ -391,41 +394,27 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT-NEXT: [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX23]], [[FOUND_CONFLICT26]]
; DEFAULT-NEXT: br i1 [[CONFLICT_RDX27]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 2
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP11]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; DEFAULT-NEXT: [[TMP4:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11:![0-9]+]]
; DEFAULT-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP5]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT31:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT30]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], [[TMP5]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP7]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[E]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE33:.*]] ]
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[D]], i64 [[INDEX]]
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; DEFAULT: [[PRED_STORE_IF]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; DEFAULT: [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF28:.*]], label %[[PRED_STORE_CONTINUE29:.*]]
-; DEFAULT: [[PRED_STORE_IF28]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE29]]
-; DEFAULT: [[PRED_STORE_CONTINUE29]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]]
-; DEFAULT: [[PRED_STORE_IF30]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE31]]
-; DEFAULT: [[PRED_STORE_CONTINUE31]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33]]
-; DEFAULT: [[PRED_STORE_IF32]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE33]]
-; DEFAULT: [[PRED_STORE_CONTINUE33]]:
-; DEFAULT-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr align 4 [[TMP16]], <4 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META19:![0-9]+]], !noalias [[META20:![0-9]+]]
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; DEFAULT-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT31]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[BROADCAST_SPLAT29]]), !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]]
+; DEFAULT-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> zeroinitializer, ptr align 4 [[TMP16]], <vscale x 4 x i1> [[BROADCAST_SPLAT29]]), !alias.scope [[META19:![0-9]+]], !noalias [[META20:![0-9]+]]
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
; DEFAULT-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
@@ -1266,37 +1255,111 @@ exit:
}
define void @predicated_store(ptr %A, ptr noalias %B, ptr noalias %C, ptr %D, ptr %E, double %divisor, i64 %loop.count) #3 {
-; COMMON-LABEL: define void @predicated_store(
-; COMMON-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], double [[DIVISOR:%.*]], i64 [[LOOP_COUNT:%.*]]) #[[ATTR3:[0-9]+]] {
-; COMMON-NEXT: [[ENTRY:.*]]:
-; COMMON-NEXT: br label %[[LOOP_HEADER:.*]]
-; COMMON: [[LOOP_HEADER]]:
-; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; COMMON-NEXT: [[GEP_B:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
-; COMMON-NEXT: [[FIRST_VAL:%.*]] = load double, ptr [[B]], align 8
-; COMMON-NEXT: store double 0.000000e+00, ptr [[D]], align 8
-; COMMON-NEXT: [[COEFF:%.*]] = load double, ptr [[C]], align 8
-; COMMON-NEXT: [[NEG:%.*]] = fneg double [[FIRST_VAL]]
-; COMMON-NEXT: [[NORM:%.*]] = fdiv double [[NEG]], [[DIVISOR]]
-; COMMON-NEXT: [[SCALED_COEFF:%.*]] = fmul double [[COEFF]], [[NORM]]
-; COMMON-NEXT: [[INPUT_VAL:%.*]] = load double, ptr [[GEP_B]], align 8
-; COMMON-NEXT: [[SCALED_INPUT:%.*]] = fmul double [[INPUT_VAL]], [[DIVISOR]]
-; COMMON-NEXT: [[PRODUCT:%.*]] = fmul double [[SCALED_COEFF]], [[SCALED_INPUT]]
-; COMMON-NEXT: [[FINAL:%.*]] = fmul double [[PRODUCT]], [[DIVISOR]]
-; COMMON-NEXT: [[IS_POS:%.*]] = fcmp ogt double [[FINAL]], 0.000000e+00
-; COMMON-NEXT: br i1 [[IS_POS]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
-; COMMON: [[THEN]]:
-; COMMON-NEXT: store double 0.000000e+00, ptr [[A]], align 8
-; COMMON-NEXT: [[GEP_E:%.*]] = getelementptr i8, ptr [[E]], i64 [[IV]]
-; COMMON-NEXT: store double 0.000000e+00, ptr [[GEP_E]], align 8
-; COMMON-NEXT: br label %[[LOOP_LATCH]]
-; COMMON: [[LOOP_LATCH]]:
-; COMMON-NEXT: store double 0.000000e+00, ptr null, align 8
-; COMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; COMMON-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[LOOP_COUNT]]
-; COMMON-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; COMMON: [[EXIT]]:
-; COMMON-NEXT: ret void
+; DEFAULT-LABEL: define void @predicated_store(
+; DEFAULT-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], double [[DIVISOR:%.*]], i64 [[LOOP_COUNT:%.*]]) #[[ATTR3:[0-9]+]] {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[LOOP_COUNT]], 1
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 6
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; DEFAULT: [[VECTOR_MEMCHECK]]:
+; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[D]], i64 8
+; DEFAULT-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8
+; DEFAULT-NEXT: [[TMP1:%.*]] = add i64 [[LOOP_COUNT]], 8
+; DEFAULT-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[E]], i64 [[TMP1]]
+; DEFAULT-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[D]], [[SCEVGEP1]]
+; DEFAULT-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; DEFAULT-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; DEFAULT-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[D]], [[SCEVGEP2]]
+; DEFAULT-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[E]], [[SCEVGEP]]
+; DEFAULT-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; DEFAULT-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; DEFAULT-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[E]], [[SCEVGEP1]]
+; DEFAULT-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; DEFAULT-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
+; DEFAULT-NEXT: br i1 [[CONFLICT_RDX9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[TMP2:%.*]] = and i64 [[TMP0]], 1
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP2]]
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[DIVISOR]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <2 x ptr> poison, ptr [[A]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT10]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE17:.*]] ]
+; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = load double, ptr [[B]], align 8
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT12]], <2 x double> poison, <2 x i32> zeroinitializer
+; DEFAULT-NEXT: store double 0.000000e+00, ptr [[D]], align 8, !alias.scope [[META31:![0-9]+]], !noalias [[META34:![0-9]+]]
+; DEFAULT-NEXT: [[TMP5:%.*]] = load double, ptr [[C]], align 8
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP6:%.*]] = fneg <2 x double> [[BROADCAST_SPLAT13]]
+; DEFAULT-NEXT: [[TMP7:%.*]] = fdiv <2 x double> [[TMP6]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT15]], [[TMP7]]
+; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; DEFAULT-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP10:%.*]] = fmul <2 x double> [[TMP8]], [[TMP9]]
+; DEFAULT-NEXT: [[TMP11:%.*]] = fmul <2 x double> [[TMP10]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP12:%.*]] = fcmp ogt <2 x double> [[TMP11]], zeroinitializer
+; DEFAULT-NEXT: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> zeroinitializer, <2 x ptr> align 8 [[BROADCAST_SPLAT11]], <2 x i1> [[TMP12]]), !alias.scope [[META37:![0-9]+]], !noalias [[META38:![0-9]+]]
+; DEFAULT-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP12]], i64 0
+; DEFAULT-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DEFAULT: [[PRED_STORE_IF]]:
+; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[E]], i64 [[INDEX]]
+; DEFAULT-NEXT: store double 0.000000e+00, ptr [[TMP14]], align 8, !alias.scope [[META38]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DEFAULT: [[PRED_STORE_CONTINUE]]:
+; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP12]], i64 1
+; DEFAULT-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17]]
+; DEFAULT: [[PRED_STORE_IF16]]:
+; DEFAULT-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 1
+; DEFAULT-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[E]], i64 [[TMP16]]
+; DEFAULT-NEXT: store double 0.000000e+00, ptr [[TMP17]], align 8, !alias.scope [[META38]]
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE17]]
+; DEFAULT: [[PRED_STORE_CONTINUE17]]:
+; DEFAULT-NEXT: store double 0.000000e+00, ptr null, align 8
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; DEFAULT-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; DEFAULT: [[SCALAR_PH]]:
+;
+; PRED-LABEL: define void @predicated_store(
+; PRED-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], double [[DIVISOR:%.*]], i64 [[LOOP_COUNT:%.*]]) #[[ATTR3:[0-9]+]] {
+; PRED-NEXT: [[ENTRY:.*]]:
+; PRED-NEXT: br label %[[LOOP_HEADER:.*]]
+; PRED: [[LOOP_HEADER]]:
+; PRED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; PRED-NEXT: [[GEP_B:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
+; PRED-NEXT: [[FIRST_VAL:%.*]] = load double, ptr [[B]], align 8
+; PRED-NEXT: store double 0.000000e+00, ptr [[D]], align 8
+; PRED-NEXT: [[COEFF:%.*]] = load double, ptr [[C]], align 8
+; PRED-NEXT: [[NEG:%.*]] = fneg double [[FIRST_VAL]]
+; PRED-NEXT: [[NORM:%.*]] = fdiv double [[NEG]], [[DIVISOR]]
+; PRED-NEXT: [[SCALED_COEFF:%.*]] = fmul double [[COEFF]], [[NORM]]
+; PRED-NEXT: [[INPUT_VAL:%.*]] = load double, ptr [[GEP_B]], align 8
+; PRED-NEXT: [[SCALED_INPUT:%.*]] = fmul double [[INPUT_VAL]], [[DIVISOR]]
+; PRED-NEXT: [[PRODUCT:%.*]] = fmul double [[SCALED_COEFF]], [[SCALED_INPUT]]
+; PRED-NEXT: [[FINAL:%.*]] = fmul double [[PRODUCT]], [[DIVISOR]]
+; PRED-NEXT: [[IS_POS:%.*]] = fcmp ogt double [[FINAL]], 0.000000e+00
+; PRED-NEXT: br i1 [[IS_POS]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; PRED: [[THEN]]:
+; PRED-NEXT: store double 0.000000e+00, ptr [[A]], align 8
+; PRED-NEXT: [[GEP_E:%.*]] = getelementptr i8, ptr [[E]], i64 [[IV]]
+; PRED-NEXT: store double 0.000000e+00, ptr [[GEP_E]], align 8
+; PRED-NEXT: br label %[[LOOP_LATCH]]
+; PRED: [[LOOP_LATCH]]:
+; PRED-NEXT: store double 0.000000e+00, ptr null, align 8
+; PRED-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; PRED-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[LOOP_COUNT]]
+; PRED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; PRED: [[EXIT]]:
+; PRED-NEXT: ret void
;
entry:
br label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
index e28a520170fa6..e66423d27819d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
@@ -11,9 +11,11 @@ define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %ds
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[SRC_2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[WIDE_LOAD]])
@@ -22,38 +24,7 @@ define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %ds
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], -1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[DST_0]], i64 [[TMP6]]
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP7]], <4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
-; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[SRC_2]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> poison, double [[TMP10]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x double> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
-; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
-; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[SRC_2]], align 8
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP14]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x double> [ [[TMP11]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
-; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[SRC_2]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP18]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x double> [ [[TMP15]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP16]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
-; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[SRC_2]], align 8
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP22]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x double> [ [[TMP19]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP20]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x double> poison)
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP7]], <4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP28:%.*]] = fcmp oeq <4 x double> [[TMP24]], zeroinitializer
; CHECK-NEXT: [[TMP29:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
index 380554dd22b64..db20bab39203e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
@@ -12,51 +12,14 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 1002)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_LOAD_CONTINUE6]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP27]]
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP3]], align 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP4]], i64 0
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 1
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP9]], align 2
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i16> [[TMP6]], i16 [[TMP10]], i64 1
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: pred.load.continue2:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i16> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 2
-; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i16, ptr [[TMP15]], align 2
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP12]], i16 [[TMP16]], i64 2
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; CHECK: pred.load.continue4:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i16> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 3
-; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
-; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = load i16, ptr [[TMP21]], align 2
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP22]], i64 3
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; CHECK: pred.load.continue6:
-; CHECK-NEXT: [[TMP24]] = phi <4 x i16> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[A:%.*]], <4 x i64> [[TMP0]]
+; CHECK-NEXT: [[TMP24]] = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> align 2 [[WIDE_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[TMP24]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP26:%.*]] = sext <4 x i16> [[TMP25]] to <4 x i32>
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[TMP27]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 356e12a532c13..e4172941a9f09 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -608,38 +608,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP49:%.*]] = sext <2 x i32> [[TMP43]] to <2 x i64>
; CHECK-NEXT: [[TMP52:%.*]] = sext <2 x i32> [[TMP44]] to <2 x i64>
; CHECK-NEXT: [[TMP55:%.*]] = sext <2 x i32> [[TMP45]] to <2 x i64>
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <2 x i64> [[TMP46]], i64 0
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP47]]
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x i64> [[TMP46]], i64 1
-; CHECK-NEXT: [[TMP59:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP48]]
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <2 x i64> [[TMP49]], i64 0
-; CHECK-NEXT: [[TMP60:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP50]]
-; CHECK-NEXT: [[TMP51:%.*]] = extractelement <2 x i64> [[TMP49]], i64 1
-; CHECK-NEXT: [[TMP61:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP51]]
-; CHECK-NEXT: [[TMP53:%.*]] = extractelement <2 x i64> [[TMP52]], i64 0
-; CHECK-NEXT: [[TMP62:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP53]]
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <2 x i64> [[TMP52]], i64 1
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP54]]
-; CHECK-NEXT: [[TMP56:%.*]] = extractelement <2 x i64> [[TMP55]], i64 0
-; CHECK-NEXT: [[TMP64:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP56]]
-; CHECK-NEXT: [[TMP57:%.*]] = extractelement <2 x i64> [[TMP55]], i64 1
-; CHECK-NEXT: [[TMP65:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP57]]
-; CHECK-NEXT: [[TMP66:%.*]] = load double, ptr [[TMP58]], align 8, !alias.scope [[META17:![0-9]+]]
-; CHECK-NEXT: [[TMP67:%.*]] = load double, ptr [[TMP59]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <2 x double> poison, double [[TMP66]], i64 0
-; CHECK-NEXT: [[TMP69:%.*]] = insertelement <2 x double> [[TMP68]], double [[TMP67]], i64 1
-; CHECK-NEXT: [[TMP70:%.*]] = load double, ptr [[TMP60]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP71:%.*]] = load double, ptr [[TMP61]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP72:%.*]] = insertelement <2 x double> poison, double [[TMP70]], i64 0
-; CHECK-NEXT: [[TMP73:%.*]] = insertelement <2 x double> [[TMP72]], double [[TMP71]], i64 1
-; CHECK-NEXT: [[TMP74:%.*]] = load double, ptr [[TMP62]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP75:%.*]] = load double, ptr [[TMP63]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <2 x double> poison, double [[TMP74]], i64 0
-; CHECK-NEXT: [[TMP77:%.*]] = insertelement <2 x double> [[TMP76]], double [[TMP75]], i64 1
-; CHECK-NEXT: [[TMP78:%.*]] = load double, ptr [[TMP64]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP79:%.*]] = load double, ptr [[TMP65]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <2 x double> poison, double [[TMP78]], i64 0
-; CHECK-NEXT: [[TMP81:%.*]] = insertelement <2 x double> [[TMP80]], double [[TMP79]], i64 1
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr double, ptr [[BASE]], <2 x i64> [[TMP46]]
+; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr double, ptr [[BASE]], <2 x i64> [[TMP49]]
+; CHECK-NEXT: [[WIDE_GEP12:%.*]] = getelementptr double, ptr [[BASE]], <2 x i64> [[TMP52]]
+; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr double, ptr [[BASE]], <2 x i64> [[TMP55]]
+; CHECK-NEXT: [[TMP69:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 8 [[WIDE_GEP]], <2 x i1> splat (i1 true), <2 x double> poison), !alias.scope [[META17:![0-9]+]]
+; CHECK-NEXT: [[TMP73:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 8 [[WIDE_GEP11]], <2 x i1> splat (i1 true), <2 x double> poison), !alias.scope [[META17]]
+; CHECK-NEXT: [[TMP77:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 8 [[WIDE_GEP12]], <2 x i1> splat (i1 true), <2 x double> poison), !alias.scope [[META17]]
+; CHECK-NEXT: [[TMP81:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 8 [[WIDE_GEP13]], <2 x i1> splat (i1 true), <2 x double> poison), !alias.scope [[META17]]
; CHECK-NEXT: [[TMP82:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP69]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP82]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP38]], align 8, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
@@ -683,14 +659,8 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP93:%.*]] = getelementptr i8, ptr [[TMP92]], i64 -8
; CHECK-NEXT: [[TMP94:%.*]] = sub nsw <2 x i32> zeroinitializer, [[VEC_IND25]]
; CHECK-NEXT: [[TMP95:%.*]] = sext <2 x i32> [[TMP94]] to <2 x i64>
-; CHECK-NEXT: [[TMP96:%.*]] = extractelement <2 x i64> [[TMP95]], i64 0
-; CHECK-NEXT: [[TMP98:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP96]]
-; CHECK-NEXT: [[TMP97:%.*]] = extractelement <2 x i64> [[TMP95]], i64 1
-; CHECK-NEXT: [[TMP99:%.*]] = getelementptr double, ptr [[BASE]], i64 [[TMP97]]
-; CHECK-NEXT: [[TMP100:%.*]] = load double, ptr [[TMP98]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP101:%.*]] = load double, ptr [[TMP99]], align 8, !alias.scope [[META17]]
-; CHECK-NEXT: [[TMP102:%.*]] = insertelement <2 x double> poison, double [[TMP100]], i64 0
-; CHECK-NEXT: [[TMP103:%.*]] = insertelement <2 x double> [[TMP102]], double [[TMP101]], i64 1
+; CHECK-NEXT: [[WIDE_GEP25:%.*]] = getelementptr double, ptr [[BASE]], <2 x i64> [[TMP95]]
+; CHECK-NEXT: [[TMP103:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 8 [[WIDE_GEP25]], <2 x i1> splat (i1 true), <2 x double> poison), !alias.scope [[META17]]
; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP103]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[INTERLEAVED_VEC26:%.*]] = shufflevector <4 x double> [[TMP104]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC26]], ptr [[TMP93]], align 8, !alias.scope [[META20]], !noalias [[META17]]
@@ -702,14 +672,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[CMP_N29:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC23]]
; CHECK-NEXT: br i1 [[CMP_N29]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL23:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL24:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL25:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL31:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL32:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL33:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL23]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL24]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL25]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL31]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL32]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL33]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[GEP_1]], i64 -8
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_2]], align 8
@@ -941,7 +911,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[TMP131:%.*]] = insertelement <8 x ptr> [[TMP130]], ptr [[TMP123]], i64 7
; CHECK-NEXT: [[INDEX_NEXT28]] = add nuw i64 [[INDEX19]], 8
; CHECK-NEXT: [[TMP132:%.*]] = icmp eq i64 [[INDEX_NEXT28]], [[N_VEC18]]
-; CHECK-NEXT: br i1 [[TMP132]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP132]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP133:%.*]] = ptrtoint <8 x ptr> [[TMP131]] to <8 x i64>
; CHECK-NEXT: [[TMP134:%.*]] = extractelement <8 x i64> [[TMP133]], i64 7
@@ -961,7 +931,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[PTR_INT:%.*]] = ptrtoint ptr [[PTR_IV_NEXT]] to i64
; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[IV]], [[COUNT]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[PTR_INT_LCSSA:%.*]] = phi i64 [ [[PTR_INT]], %[[LOOP]] ], [ [[TMP87]], %[[MIDDLE_BLOCK]] ], [ [[TMP134]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[PTR_INT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 9b37d5280e262..21069503e2d8b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -225,26 +225,14 @@ define void @main_vector_loop_fixed_single_vector_iteration_with_runtime_checks(
; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-NOTF: [[VECTOR_BODY]]:
; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 2
-; CHECK-NOTF-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], 4
-; CHECK-NOTF-NEXT: [[TMP3:%.*]] = add i64 [[TMP0]], 6
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[J]], i64 [[TMP0]]
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NOTF-NEXT: [[TMP5:%.*]] = trunc <4 x i64> [[STRIDED_VEC]] to <4 x i16>
-; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP0]]
-; CHECK-NOTF-NEXT: [[TMP11:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP1]]
-; CHECK-NOTF-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP2]]
-; CHECK-NOTF-NEXT: [[TMP13:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP3]]
-; CHECK-NOTF-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[TMP5]], i64 0
-; CHECK-NOTF-NEXT: store i16 [[TMP6]], ptr [[TMP10]], align 2
-; CHECK-NOTF-NEXT: [[TMP7:%.*]] = extractelement <4 x i16> [[TMP5]], i64 1
-; CHECK-NOTF-NEXT: store i16 [[TMP7]], ptr [[TMP11]], align 2
-; CHECK-NOTF-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 2
-; CHECK-NOTF-NEXT: store i16 [[TMP8]], ptr [[TMP12]], align 2
-; CHECK-NOTF-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
-; CHECK-NOTF-NEXT: store i16 [[TMP9]], ptr [[TMP13]], align 2
+; CHECK-NOTF-NEXT: [[WIDE_GEP:%.*]] = getelementptr i16, ptr [[K]], <4 x i64> [[VEC_IND]]
+; CHECK-NOTF-NEXT: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> [[TMP5]], <4 x ptr> align 2 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NOTF-NEXT: store i64 0, ptr [[A]], align 8
; CHECK-NOTF-NEXT: store i64 0, ptr [[B]], align 8
; CHECK-NOTF-NEXT: store i64 0, ptr [[C]], align 8
@@ -256,6 +244,7 @@ define void @main_vector_loop_fixed_single_vector_iteration_with_runtime_checks(
; CHECK-NOTF-NEXT: store i64 0, ptr [[I]], align 8
; CHECK-NOTF-NEXT: store i64 0, ptr [[L]], align 8
; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NOTF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
; CHECK-NOTF-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
; CHECK-NOTF-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-NOTF: [[MIDDLE_BLOCK]]:
@@ -290,24 +279,14 @@ define void @main_vector_loop_fixed_single_vector_iteration_with_runtime_checks(
; CHECK-TF-NEXT: [[ENTRY:.*:]]
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
-; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
-; CHECK-TF-NEXT: [[TMP3:%.*]] = mul <vscale x 8 x i64> [[TMP2]], splat (i64 2)
-; CHECK-TF-NEXT: [[TMP4:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 8)
; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-TF: [[VECTOR_BODY]]:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[J]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i64> @llvm.masked.gather.nxv8i64.nxv8p0(<vscale x 8 x ptr> align 8 [[WIDE_GEP]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i64> poison)
-; CHECK-TF-NEXT: [[TMP5:%.*]] = trunc <vscale x 8 x i64> [[WIDE_MASKED_GATHER]] to <vscale x 8 x i16>
-; CHECK-TF-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i16, ptr [[K]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-TF-NEXT: call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> [[TMP5]], <vscale x 8 x ptr> align 2 [[WIDE_GEP1]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[J]], <8 x i64> <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14>
+; CHECK-TF-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> align 8 [[WIDE_GEP]], <8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], <8 x i64> poison)
+; CHECK-TF-NEXT: [[TMP0:%.*]] = trunc <8 x i64> [[WIDE_MASKED_GATHER]] to <8 x i16>
+; CHECK-TF-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i16, ptr [[K]], <8 x i64> <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14>
+; CHECK-TF-NEXT: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> [[TMP0]], <8 x ptr> align 2 [[WIDE_GEP1]], <8 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
; CHECK-TF-NEXT: store i64 0, ptr [[A]], align 8
; CHECK-TF-NEXT: store i64 0, ptr [[B]], align 8
; CHECK-TF-NEXT: store i64 0, ptr [[C]], align 8
@@ -318,12 +297,7 @@ define void @main_vector_loop_fixed_single_vector_iteration_with_runtime_checks(
; CHECK-TF-NEXT: store i64 0, ptr [[H]], align 8
; CHECK-TF-NEXT: store i64 0, ptr [[I]], align 8
; CHECK-TF-NEXT: store i64 0, ptr [[L]], align 8
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 8)
-; CHECK-TF-NEXT: [[TMP6:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
-; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br label %[[EXIT:.*]]
; CHECK-TF: [[EXIT]]:
@@ -527,7 +501,7 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TF-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; CHECK-TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]])
; CHECK-TF-NEXT: br label %[[EXIT:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index a9e8233615783..e411789f601a3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -402,77 +402,18 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP67:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 8
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 9
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 10
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 11
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 12
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 13
-; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 14
-; CHECK-INTERLEAVE1-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 15
; CHECK-INTERLEAVE1-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1
; CHECK-INTERLEAVE1-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP13]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP14]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP33:%.*]] = load i16, ptr [[TMP17]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP34:%.*]] = load i16, ptr [[TMP18]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP35:%.*]] = load i16, ptr [[TMP19]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP36:%.*]] = load i16, ptr [[TMP20]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP37:%.*]] = load i16, ptr [[TMP21]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP38:%.*]] = load i16, ptr [[TMP22]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP39:%.*]] = load i16, ptr [[TMP23]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP40:%.*]] = load i16, ptr [[TMP24]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP41:%.*]] = load i16, ptr [[TMP25]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP42:%.*]] = load i16, ptr [[TMP26]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP43:%.*]] = load i16, ptr [[TMP27]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP44:%.*]] = load i16, ptr [[TMP28]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP45:%.*]] = load i16, ptr [[TMP29]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP46:%.*]] = load i16, ptr [[TMP30]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP47:%.*]] = load i16, ptr [[TMP31]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP48:%.*]] = load i16, ptr [[TMP32]], align 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP49:%.*]] = insertelement <16 x i16> poison, i16 [[TMP33]], i64 0
-; CHECK-INTERLEAVE1-NEXT: [[TMP50:%.*]] = insertelement <16 x i16> [[TMP49]], i16 [[TMP34]], i64 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP51:%.*]] = insertelement <16 x i16> [[TMP50]], i16 [[TMP35]], i64 2
-; CHECK-INTERLEAVE1-NEXT: [[TMP52:%.*]] = insertelement <16 x i16> [[TMP51]], i16 [[TMP36]], i64 3
-; CHECK-INTERLEAVE1-NEXT: [[TMP53:%.*]] = insertelement <16 x i16> [[TMP52]], i16 [[TMP37]], i64 4
-; CHECK-INTERLEAVE1-NEXT: [[TMP54:%.*]] = insertelement <16 x i16> [[TMP53]], i16 [[TMP38]], i64 5
-; CHECK-INTERLEAVE1-NEXT: [[TMP55:%.*]] = insertelement <16 x i16> [[TMP54]], i16 [[TMP39]], i64 6
-; CHECK-INTERLEAVE1-NEXT: [[TMP56:%.*]] = insertelement <16 x i16> [[TMP55]], i16 [[TMP40]], i64 7
-; CHECK-INTERLEAVE1-NEXT: [[TMP57:%.*]] = insertelement <16 x i16> [[TMP56]], i16 [[TMP41]], i64 8
-; CHECK-INTERLEAVE1-NEXT: [[TMP58:%.*]] = insertelement <16 x i16> [[TMP57]], i16 [[TMP42]], i64 9
-; CHECK-INTERLEAVE1-NEXT: [[TMP59:%.*]] = insertelement <16 x i16> [[TMP58]], i16 [[TMP43]], i64 10
-; CHECK-INTERLEAVE1-NEXT: [[TMP60:%.*]] = insertelement <16 x i16> [[TMP59]], i16 [[TMP44]], i64 11
-; CHECK-INTERLEAVE1-NEXT: [[TMP61:%.*]] = insertelement <16 x i16> [[TMP60]], i16 [[TMP45]], i64 12
-; CHECK-INTERLEAVE1-NEXT: [[TMP62:%.*]] = insertelement <16 x i16> [[TMP61]], i16 [[TMP46]], i64 13
-; CHECK-INTERLEAVE1-NEXT: [[TMP63:%.*]] = insertelement <16 x i16> [[TMP62]], i16 [[TMP47]], i64 14
-; CHECK-INTERLEAVE1-NEXT: [[TMP64:%.*]] = insertelement <16 x i16> [[TMP63]], i16 [[TMP48]], i64 15
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[B]], <16 x i64> [[VEC_IND]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP64:%.*]] = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> align 2 [[WIDE_GEP]], <16 x i1> splat (i1 true), <16 x i16> poison)
; CHECK-INTERLEAVE1-NEXT: [[TMP65:%.*]] = zext <16 x i16> [[TMP64]] to <16 x i32>
; CHECK-INTERLEAVE1-NEXT: [[TMP66:%.*]] = mul <16 x i32> [[TMP65]], [[TMP16]]
; CHECK-INTERLEAVE1-NEXT: [[TMP67]] = add <16 x i32> [[TMP66]], [[VEC_PHI]]
; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT: [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
; CHECK-INTERLEAVE1-NEXT: [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP68]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
@@ -489,153 +430,22 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP135:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP136:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 8
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 9
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 10
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 11
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 12
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 13
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 14
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 15
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 16
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 17
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 18
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 19
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 20
-; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 21
-; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 22
-; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], 23
-; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 24
-; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 25
-; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 26
-; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = add i64 [[INDEX]], 27
-; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = add i64 [[INDEX]], 28
-; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = add i64 [[INDEX]], 29
-; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = add i64 [[INDEX]], 30
-; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = add i64 [[INDEX]], 31
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP31]], i64 16
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP31]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP32]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP31]], align 1
; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP40:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
-; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
-; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
-; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
-; CHECK-INTERLEAVED-NEXT: [[TMP45:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; CHECK-INTERLEAVED-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
-; CHECK-INTERLEAVED-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
-; CHECK-INTERLEAVED-NEXT: [[TMP48:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
-; CHECK-INTERLEAVED-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP13]]
-; CHECK-INTERLEAVED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP14]]
-; CHECK-INTERLEAVED-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP15]]
-; CHECK-INTERLEAVED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP16]]
-; CHECK-INTERLEAVED-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP17]]
-; CHECK-INTERLEAVED-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP18]]
-; CHECK-INTERLEAVED-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP19]]
-; CHECK-INTERLEAVED-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP20]]
-; CHECK-INTERLEAVED-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP21]]
-; CHECK-INTERLEAVED-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP22]]
-; CHECK-INTERLEAVED-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP23]]
-; CHECK-INTERLEAVED-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP24]]
-; CHECK-INTERLEAVED-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP25]]
-; CHECK-INTERLEAVED-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP26]]
-; CHECK-INTERLEAVED-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP27]]
-; CHECK-INTERLEAVED-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP28]]
-; CHECK-INTERLEAVED-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP29]]
-; CHECK-INTERLEAVED-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP30]]
-; CHECK-INTERLEAVED-NEXT: [[TMP67:%.*]] = load i16, ptr [[TMP35]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP68:%.*]] = load i16, ptr [[TMP36]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP69:%.*]] = load i16, ptr [[TMP37]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP70:%.*]] = load i16, ptr [[TMP38]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP71:%.*]] = load i16, ptr [[TMP39]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP72:%.*]] = load i16, ptr [[TMP40]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP73:%.*]] = load i16, ptr [[TMP41]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP74:%.*]] = load i16, ptr [[TMP42]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP75:%.*]] = load i16, ptr [[TMP43]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP76:%.*]] = load i16, ptr [[TMP44]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP77:%.*]] = load i16, ptr [[TMP45]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP78:%.*]] = load i16, ptr [[TMP46]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP79:%.*]] = load i16, ptr [[TMP47]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP80:%.*]] = load i16, ptr [[TMP48]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP81:%.*]] = load i16, ptr [[TMP49]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP82:%.*]] = load i16, ptr [[TMP50]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP83:%.*]] = insertelement <16 x i16> poison, i16 [[TMP67]], i64 0
-; CHECK-INTERLEAVED-NEXT: [[TMP84:%.*]] = insertelement <16 x i16> [[TMP83]], i16 [[TMP68]], i64 1
-; CHECK-INTERLEAVED-NEXT: [[TMP85:%.*]] = insertelement <16 x i16> [[TMP84]], i16 [[TMP69]], i64 2
-; CHECK-INTERLEAVED-NEXT: [[TMP86:%.*]] = insertelement <16 x i16> [[TMP85]], i16 [[TMP70]], i64 3
-; CHECK-INTERLEAVED-NEXT: [[TMP87:%.*]] = insertelement <16 x i16> [[TMP86]], i16 [[TMP71]], i64 4
-; CHECK-INTERLEAVED-NEXT: [[TMP88:%.*]] = insertelement <16 x i16> [[TMP87]], i16 [[TMP72]], i64 5
-; CHECK-INTERLEAVED-NEXT: [[TMP89:%.*]] = insertelement <16 x i16> [[TMP88]], i16 [[TMP73]], i64 6
-; CHECK-INTERLEAVED-NEXT: [[TMP90:%.*]] = insertelement <16 x i16> [[TMP89]], i16 [[TMP74]], i64 7
-; CHECK-INTERLEAVED-NEXT: [[TMP91:%.*]] = insertelement <16 x i16> [[TMP90]], i16 [[TMP75]], i64 8
-; CHECK-INTERLEAVED-NEXT: [[TMP92:%.*]] = insertelement <16 x i16> [[TMP91]], i16 [[TMP76]], i64 9
-; CHECK-INTERLEAVED-NEXT: [[TMP93:%.*]] = insertelement <16 x i16> [[TMP92]], i16 [[TMP77]], i64 10
-; CHECK-INTERLEAVED-NEXT: [[TMP94:%.*]] = insertelement <16 x i16> [[TMP93]], i16 [[TMP78]], i64 11
-; CHECK-INTERLEAVED-NEXT: [[TMP95:%.*]] = insertelement <16 x i16> [[TMP94]], i16 [[TMP79]], i64 12
-; CHECK-INTERLEAVED-NEXT: [[TMP96:%.*]] = insertelement <16 x i16> [[TMP95]], i16 [[TMP80]], i64 13
-; CHECK-INTERLEAVED-NEXT: [[TMP97:%.*]] = insertelement <16 x i16> [[TMP96]], i16 [[TMP81]], i64 14
-; CHECK-INTERLEAVED-NEXT: [[TMP98:%.*]] = insertelement <16 x i16> [[TMP97]], i16 [[TMP82]], i64 15
-; CHECK-INTERLEAVED-NEXT: [[TMP99:%.*]] = load i16, ptr [[TMP51]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP100:%.*]] = load i16, ptr [[TMP52]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP101:%.*]] = load i16, ptr [[TMP53]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP102:%.*]] = load i16, ptr [[TMP54]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP103:%.*]] = load i16, ptr [[TMP55]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP104:%.*]] = load i16, ptr [[TMP56]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP105:%.*]] = load i16, ptr [[TMP57]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP106:%.*]] = load i16, ptr [[TMP58]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP107:%.*]] = load i16, ptr [[TMP59]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP108:%.*]] = load i16, ptr [[TMP60]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP109:%.*]] = load i16, ptr [[TMP61]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP110:%.*]] = load i16, ptr [[TMP62]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP111:%.*]] = load i16, ptr [[TMP63]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP112:%.*]] = load i16, ptr [[TMP64]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP113:%.*]] = load i16, ptr [[TMP65]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP114:%.*]] = load i16, ptr [[TMP66]], align 2
-; CHECK-INTERLEAVED-NEXT: [[TMP115:%.*]] = insertelement <16 x i16> poison, i16 [[TMP99]], i64 0
-; CHECK-INTERLEAVED-NEXT: [[TMP116:%.*]] = insertelement <16 x i16> [[TMP115]], i16 [[TMP100]], i64 1
-; CHECK-INTERLEAVED-NEXT: [[TMP117:%.*]] = insertelement <16 x i16> [[TMP116]], i16 [[TMP101]], i64 2
-; CHECK-INTERLEAVED-NEXT: [[TMP118:%.*]] = insertelement <16 x i16> [[TMP117]], i16 [[TMP102]], i64 3
-; CHECK-INTERLEAVED-NEXT: [[TMP119:%.*]] = insertelement <16 x i16> [[TMP118]], i16 [[TMP103]], i64 4
-; CHECK-INTERLEAVED-NEXT: [[TMP120:%.*]] = insertelement <16 x i16> [[TMP119]], i16 [[TMP104]], i64 5
-; CHECK-INTERLEAVED-NEXT: [[TMP121:%.*]] = insertelement <16 x i16> [[TMP120]], i16 [[TMP105]], i64 6
-; CHECK-INTERLEAVED-NEXT: [[TMP122:%.*]] = insertelement <16 x i16> [[TMP121]], i16 [[TMP106]], i64 7
-; CHECK-INTERLEAVED-NEXT: [[TMP123:%.*]] = insertelement <16 x i16> [[TMP122]], i16 [[TMP107]], i64 8
-; CHECK-INTERLEAVED-NEXT: [[TMP124:%.*]] = insertelement <16 x i16> [[TMP123]], i16 [[TMP108]], i64 9
-; CHECK-INTERLEAVED-NEXT: [[TMP125:%.*]] = insertelement <16 x i16> [[TMP124]], i16 [[TMP109]], i64 10
-; CHECK-INTERLEAVED-NEXT: [[TMP126:%.*]] = insertelement <16 x i16> [[TMP125]], i16 [[TMP110]], i64 11
-; CHECK-INTERLEAVED-NEXT: [[TMP127:%.*]] = insertelement <16 x i16> [[TMP126]], i16 [[TMP111]], i64 12
-; CHECK-INTERLEAVED-NEXT: [[TMP128:%.*]] = insertelement <16 x i16> [[TMP127]], i16 [[TMP112]], i64 13
-; CHECK-INTERLEAVED-NEXT: [[TMP129:%.*]] = insertelement <16 x i16> [[TMP128]], i16 [[TMP113]], i64 14
-; CHECK-INTERLEAVED-NEXT: [[TMP130:%.*]] = insertelement <16 x i16> [[TMP129]], i16 [[TMP114]], i64 15
-; CHECK-INTERLEAVED-NEXT: [[TMP131:%.*]] = zext <16 x i16> [[TMP98]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[B]], <16 x i64> [[VEC_IND]]
+; CHECK-INTERLEAVED-NEXT: [[TMP130:%.*]] = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> align 2 [[WIDE_GEP]], <16 x i1> splat (i1 true), <16 x i16> poison)
; CHECK-INTERLEAVED-NEXT: [[TMP132:%.*]] = zext <16 x i16> [[TMP130]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP133:%.*]] = mul <16 x i32> [[TMP131]], [[TMP33]]
; CHECK-INTERLEAVED-NEXT: [[TMP134:%.*]] = mul <16 x i32> [[TMP132]], [[TMP34]]
-; CHECK-INTERLEAVED-NEXT: [[TMP135]] = add <16 x i32> [[TMP133]], [[VEC_PHI]]
; CHECK-INTERLEAVED-NEXT: [[TMP136]] = add <16 x i32> [[TMP134]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT: [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
; CHECK-INTERLEAVED-NEXT: [[TMP137:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP137]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP136]], [[TMP135]]
-; CHECK-INTERLEAVED-NEXT: [[TMP138:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP138:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP136]])
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_EXIT:.*]]
; CHECK-INTERLEAVED: [[FOR_EXIT]]:
; CHECK-INTERLEAVED-NEXT: ret i32 [[TMP138]]
@@ -648,77 +458,18 @@ define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-MAXBW: [[VECTOR_BODY]]:
; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP67:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 5
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 6
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 7
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 8
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 9
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 10
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 11
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 12
-; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 13
-; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 14
-; CHECK-MAXBW-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 15
; CHECK-MAXBW-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1
; CHECK-MAXBW-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-MAXBW-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-MAXBW-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-MAXBW-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; CHECK-MAXBW-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
-; CHECK-MAXBW-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; CHECK-MAXBW-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
-; CHECK-MAXBW-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
-; CHECK-MAXBW-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
-; CHECK-MAXBW-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; CHECK-MAXBW-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
-; CHECK-MAXBW-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
-; CHECK-MAXBW-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
-; CHECK-MAXBW-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP13]]
-; CHECK-MAXBW-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP14]]
-; CHECK-MAXBW-NEXT: [[TMP33:%.*]] = load i16, ptr [[TMP17]], align 2
-; CHECK-MAXBW-NEXT: [[TMP34:%.*]] = load i16, ptr [[TMP18]], align 2
-; CHECK-MAXBW-NEXT: [[TMP35:%.*]] = load i16, ptr [[TMP19]], align 2
-; CHECK-MAXBW-NEXT: [[TMP36:%.*]] = load i16, ptr [[TMP20]], align 2
-; CHECK-MAXBW-NEXT: [[TMP37:%.*]] = load i16, ptr [[TMP21]], align 2
-; CHECK-MAXBW-NEXT: [[TMP38:%.*]] = load i16, ptr [[TMP22]], align 2
-; CHECK-MAXBW-NEXT: [[TMP39:%.*]] = load i16, ptr [[TMP23]], align 2
-; CHECK-MAXBW-NEXT: [[TMP40:%.*]] = load i16, ptr [[TMP24]], align 2
-; CHECK-MAXBW-NEXT: [[TMP41:%.*]] = load i16, ptr [[TMP25]], align 2
-; CHECK-MAXBW-NEXT: [[TMP42:%.*]] = load i16, ptr [[TMP26]], align 2
-; CHECK-MAXBW-NEXT: [[TMP43:%.*]] = load i16, ptr [[TMP27]], align 2
-; CHECK-MAXBW-NEXT: [[TMP44:%.*]] = load i16, ptr [[TMP28]], align 2
-; CHECK-MAXBW-NEXT: [[TMP45:%.*]] = load i16, ptr [[TMP29]], align 2
-; CHECK-MAXBW-NEXT: [[TMP46:%.*]] = load i16, ptr [[TMP30]], align 2
-; CHECK-MAXBW-NEXT: [[TMP47:%.*]] = load i16, ptr [[TMP31]], align 2
-; CHECK-MAXBW-NEXT: [[TMP48:%.*]] = load i16, ptr [[TMP32]], align 2
-; CHECK-MAXBW-NEXT: [[TMP49:%.*]] = insertelement <16 x i16> poison, i16 [[TMP33]], i64 0
-; CHECK-MAXBW-NEXT: [[TMP50:%.*]] = insertelement <16 x i16> [[TMP49]], i16 [[TMP34]], i64 1
-; CHECK-MAXBW-NEXT: [[TMP51:%.*]] = insertelement <16 x i16> [[TMP50]], i16 [[TMP35]], i64 2
-; CHECK-MAXBW-NEXT: [[TMP52:%.*]] = insertelement <16 x i16> [[TMP51]], i16 [[TMP36]], i64 3
-; CHECK-MAXBW-NEXT: [[TMP53:%.*]] = insertelement <16 x i16> [[TMP52]], i16 [[TMP37]], i64 4
-; CHECK-MAXBW-NEXT: [[TMP54:%.*]] = insertelement <16 x i16> [[TMP53]], i16 [[TMP38]], i64 5
-; CHECK-MAXBW-NEXT: [[TMP55:%.*]] = insertelement <16 x i16> [[TMP54]], i16 [[TMP39]], i64 6
-; CHECK-MAXBW-NEXT: [[TMP56:%.*]] = insertelement <16 x i16> [[TMP55]], i16 [[TMP40]], i64 7
-; CHECK-MAXBW-NEXT: [[TMP57:%.*]] = insertelement <16 x i16> [[TMP56]], i16 [[TMP41]], i64 8
-; CHECK-MAXBW-NEXT: [[TMP58:%.*]] = insertelement <16 x i16> [[TMP57]], i16 [[TMP42]], i64 9
-; CHECK-MAXBW-NEXT: [[TMP59:%.*]] = insertelement <16 x i16> [[TMP58]], i16 [[TMP43]], i64 10
-; CHECK-MAXBW-NEXT: [[TMP60:%.*]] = insertelement <16 x i16> [[TMP59]], i16 [[TMP44]], i64 11
-; CHECK-MAXBW-NEXT: [[TMP61:%.*]] = insertelement <16 x i16> [[TMP60]], i16 [[TMP45]], i64 12
-; CHECK-MAXBW-NEXT: [[TMP62:%.*]] = insertelement <16 x i16> [[TMP61]], i16 [[TMP46]], i64 13
-; CHECK-MAXBW-NEXT: [[TMP63:%.*]] = insertelement <16 x i16> [[TMP62]], i16 [[TMP47]], i64 14
-; CHECK-MAXBW-NEXT: [[TMP64:%.*]] = insertelement <16 x i16> [[TMP63]], i16 [[TMP48]], i64 15
+; CHECK-MAXBW-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[B]], <16 x i64> [[VEC_IND]]
+; CHECK-MAXBW-NEXT: [[TMP64:%.*]] = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> align 2 [[WIDE_GEP]], <16 x i1> splat (i1 true), <16 x i16> poison)
; CHECK-MAXBW-NEXT: [[TMP65:%.*]] = zext <16 x i16> [[TMP64]] to <16 x i32>
; CHECK-MAXBW-NEXT: [[TMP66:%.*]] = mul <16 x i32> [[TMP65]], [[TMP16]]
; CHECK-MAXBW-NEXT: [[TMP67]] = add <16 x i32> [[TMP66]], [[VEC_PHI]]
; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-MAXBW-NEXT: [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 16)
; CHECK-MAXBW-NEXT: [[TMP68:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-MAXBW-NEXT: br i1 [[TMP68]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 533240f48967f..36374caf0ce52 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -64,45 +64,28 @@ define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8
; CHECK-NEXT: [[BOUND113:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP5]]
; CHECK-NEXT: [[FOUND_CONFLICT14:%.*]] = and i1 [[BOUND012]], [[BOUND113]]
; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i8> poison, i8 [[TMP27]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i8> [[BROADCAST_SPLATINSERT]], <2 x i8> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = zext <2 x i8> [[BROADCAST_SPLAT]] to <2 x i64>
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 0, i32 [[TMP2]])
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE22:.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = zext <2 x i8> [[VEC_IND]] to <2 x i64>
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
-; CHECK-NEXT: br i1 [[TMP37]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
-; CHECK: [[PRED_STORE_IF19]]:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <2 x i64> [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP115:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP114]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP116:%.*]] = extractelement <2 x i64> [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP117:%.*]] = or i64 [[TMP116]], 1
-; CHECK-NEXT: store i64 [[TMP117]], ptr [[TMP115]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]
-; CHECK: [[PRED_STORE_CONTINUE20]]:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
-; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[PRED_STORE_IF21:.*]]
; CHECK: [[PRED_STORE_IF21]]:
-; CHECK-NEXT: [[TMP120:%.*]] = extractelement <2 x i64> [[TMP26]], i64 1
-; CHECK-NEXT: [[TMP121:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP120]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <2 x i64> [[TMP25]], i64 1
-; CHECK-NEXT: [[TMP123:%.*]] = or i64 [[TMP122]], 1
-; CHECK-NEXT: store i64 [[TMP123]], ptr [[TMP121]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP25]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i64>
+; CHECK-NEXT: [[TMP27:%.*]] = or <4 x i64> [[TMP26]], splat (i64 1)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22:.*]]
; CHECK: [[PRED_STORE_CONTINUE22]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[PRED_STORE_IF21]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[PRED_STORE_IF21]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 4, i8 8, i8 12>, %[[PRED_STORE_IF21]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = zext <4 x i8> [[VEC_IND]] to <4 x i64>
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], <4 x i64> [[TMP28]], i64 [[OFF]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP27]], <4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP48:%.*]] = xor i1 [[TMP47]], true
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 8)
-; CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 16)
+; CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE22]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: store i8 0, ptr [[DST_2]], align 1, !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -208,74 +191,10 @@ exit:
define void @srem_sdiv_without_tail_folding(i32 %d.0, i32 %d.1, ptr %dst, i32 %end) #1 {
; CHECK-LABEL: define void @srem_sdiv_without_tail_folding(
; CHECK-SAME: i32 [[D_0:%.*]], i32 [[D_1:%.*]], ptr [[DST:%.*]], i32 [[END:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[END]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[END]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[END]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[D_0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[D_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -1)
-; CHECK-NEXT: [[TMP1:%.*]] = srem <4 x i32> [[TMP0]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], splat (i32 1)
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP23:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> [[TMP0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i1> [[TMP3]])
-; CHECK-NEXT: [[TMP24:%.*]] = add <4 x i32> [[TMP23]], splat (i32 1)
-; CHECK-NEXT: [[TMP25:%.*]] = sext <4 x i32> [[TMP24]] to <4 x i64>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]
-; CHECK-NEXT: store i32 [[INDEX]], ptr [[TMP28]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
-; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i64> [[TMP25]], i64 1
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
-; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i64> [[TMP25]], i64 2
-; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: store i32 [[TMP37]], ptr [[TMP36]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
-; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP39:%.*]] = extractelement <4 x i64> [[TMP25]], i64 3
-; CHECK-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP39]]
-; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[INDEX]], 3
-; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP40]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[END]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[IV_SUB:%.*]] = add nsw i32 [[IV]], -1
; CHECK-NEXT: [[REM:%.*]] = srem i32 [[IV_SUB]], [[D_0]]
; CHECK-NEXT: [[REM_1:%.*]] = add nsw i32 [[REM]], 1
@@ -291,7 +210,7 @@ define void @srem_sdiv_without_tail_folding(i32 %d.0, i32 %d.1, ptr %dst, i32 %e
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[END]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -390,7 +309,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.sin.v4f64(<4 x double> [[TMP2]])
@@ -416,7 +335,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: store float [[PHI]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -457,9 +376,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF17:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF15:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF17]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF15]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -513,7 +432,5 @@ exit:
; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META6]], [[META7]]}
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META7]], [[META8]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META7]]}
-; CHECK: [[PROF17]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[PROF15]] = !{!"branch_weights", i32 0, i32 1}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 132d47a12a7c8..22e0eca927a66 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -32,22 +32,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; DEFAULT-NEXT: [[TMP8:%.*]] = and <4 x i32> [[BROADCAST_SPLAT2]], splat (i32 1)
; DEFAULT-NEXT: [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], splat (i32 1)
; DEFAULT-NEXT: [[TMP10:%.*]] = zext <4 x i32> [[TMP9]] to <4 x i64>
-; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
-; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP11]]
-; DEFAULT-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP10]], i64 1
-; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP13]]
-; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP10]], i64 2
-; DEFAULT-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP15]]
-; DEFAULT-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP10]], i64 3
-; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP17]]
-; DEFAULT-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP12]], align 4
-; DEFAULT-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP14]], align 4
-; DEFAULT-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP16]], align 4
-; DEFAULT-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 4
-; DEFAULT-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i64 0
-; DEFAULT-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i64 1
-; DEFAULT-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i64 2
-; DEFAULT-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i64 3
+; DEFAULT-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP10]]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
@@ -77,72 +62,27 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; DEFAULT-NEXT: [[TMP44:%.*]] = or <4 x i32> [[TMP41]], [[BROADCAST_SPLAT]]
; DEFAULT-NEXT: [[TMP45:%.*]] = or <4 x i32> [[TMP42]], [[BROADCAST_SPLAT]]
; DEFAULT-NEXT: [[TMP46:%.*]] = or <4 x i32> [[TMP43]], [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT: [[TMP26:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[WIDE_MASKED_GATHER11:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; DEFAULT-NEXT: [[TMP47:%.*]] = lshr <4 x i32> [[TMP44]], splat (i32 1)
; DEFAULT-NEXT: [[TMP48:%.*]] = lshr <4 x i32> [[TMP45]], splat (i32 1)
; DEFAULT-NEXT: [[TMP49:%.*]] = lshr <4 x i32> [[TMP46]], splat (i32 1)
; DEFAULT-NEXT: [[TMP50:%.*]] = zext <4 x i32> [[TMP47]] to <4 x i64>
; DEFAULT-NEXT: [[TMP51:%.*]] = zext <4 x i32> [[TMP48]] to <4 x i64>
; DEFAULT-NEXT: [[TMP52:%.*]] = zext <4 x i32> [[TMP49]] to <4 x i64>
-; DEFAULT-NEXT: [[TMP53:%.*]] = extractelement <4 x i64> [[TMP50]], i64 0
-; DEFAULT-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP53]]
-; DEFAULT-NEXT: [[TMP55:%.*]] = extractelement <4 x i64> [[TMP50]], i64 1
-; DEFAULT-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP55]]
-; DEFAULT-NEXT: [[TMP57:%.*]] = extractelement <4 x i64> [[TMP50]], i64 2
-; DEFAULT-NEXT: [[TMP58:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP57]]
-; DEFAULT-NEXT: [[TMP59:%.*]] = extractelement <4 x i64> [[TMP50]], i64 3
-; DEFAULT-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP59]]
-; DEFAULT-NEXT: [[TMP61:%.*]] = extractelement <4 x i64> [[TMP51]], i64 0
-; DEFAULT-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP61]]
-; DEFAULT-NEXT: [[TMP63:%.*]] = extractelement <4 x i64> [[TMP51]], i64 1
-; DEFAULT-NEXT: [[TMP64:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP63]]
-; DEFAULT-NEXT: [[TMP65:%.*]] = extractelement <4 x i64> [[TMP51]], i64 2
-; DEFAULT-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP65]]
-; DEFAULT-NEXT: [[TMP120:%.*]] = extractelement <4 x i64> [[TMP51]], i64 3
-; DEFAULT-NEXT: [[TMP121:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP120]]
-; DEFAULT-NEXT: [[TMP122:%.*]] = extractelement <4 x i64> [[TMP52]], i64 0
-; DEFAULT-NEXT: [[TMP123:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP122]]
-; DEFAULT-NEXT: [[TMP124:%.*]] = extractelement <4 x i64> [[TMP52]], i64 1
-; DEFAULT-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP124]]
-; DEFAULT-NEXT: [[TMP126:%.*]] = extractelement <4 x i64> [[TMP52]], i64 2
-; DEFAULT-NEXT: [[TMP127:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP126]]
-; DEFAULT-NEXT: [[TMP128:%.*]] = extractelement <4 x i64> [[TMP52]], i64 3
-; DEFAULT-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP128]]
-; DEFAULT-NEXT: [[TMP77:%.*]] = load i32, ptr [[TMP54]], align 4
-; DEFAULT-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP56]], align 4
-; DEFAULT-NEXT: [[TMP79:%.*]] = load i32, ptr [[TMP58]], align 4
-; DEFAULT-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP60]], align 4
-; DEFAULT-NEXT: [[TMP81:%.*]] = insertelement <4 x i32> poison, i32 [[TMP77]], i64 0
-; DEFAULT-NEXT: [[TMP82:%.*]] = insertelement <4 x i32> [[TMP81]], i32 [[TMP78]], i64 1
-; DEFAULT-NEXT: [[TMP83:%.*]] = insertelement <4 x i32> [[TMP82]], i32 [[TMP79]], i64 2
-; DEFAULT-NEXT: [[TMP84:%.*]] = insertelement <4 x i32> [[TMP83]], i32 [[TMP80]], i64 3
-; DEFAULT-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP62]], align 4
-; DEFAULT-NEXT: [[TMP86:%.*]] = load i32, ptr [[TMP64]], align 4
-; DEFAULT-NEXT: [[TMP87:%.*]] = load i32, ptr [[TMP66]], align 4
-; DEFAULT-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP121]], align 4
-; DEFAULT-NEXT: [[TMP89:%.*]] = insertelement <4 x i32> poison, i32 [[TMP85]], i64 0
-; DEFAULT-NEXT: [[TMP90:%.*]] = insertelement <4 x i32> [[TMP89]], i32 [[TMP86]], i64 1
-; DEFAULT-NEXT: [[TMP91:%.*]] = insertelement <4 x i32> [[TMP90]], i32 [[TMP87]], i64 2
-; DEFAULT-NEXT: [[TMP92:%.*]] = insertelement <4 x i32> [[TMP91]], i32 [[TMP88]], i64 3
-; DEFAULT-NEXT: [[TMP93:%.*]] = load i32, ptr [[TMP123]], align 4
-; DEFAULT-NEXT: [[TMP94:%.*]] = load i32, ptr [[TMP125]], align 4
-; DEFAULT-NEXT: [[TMP95:%.*]] = load i32, ptr [[TMP127]], align 4
-; DEFAULT-NEXT: [[TMP96:%.*]] = load i32, ptr [[TMP76]], align 4
-; DEFAULT-NEXT: [[TMP97:%.*]] = insertelement <4 x i32> poison, i32 [[TMP93]], i64 0
-; DEFAULT-NEXT: [[TMP98:%.*]] = insertelement <4 x i32> [[TMP97]], i32 [[TMP94]], i64 1
-; DEFAULT-NEXT: [[TMP99:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP95]], i64 2
-; DEFAULT-NEXT: [[TMP100:%.*]] = insertelement <4 x i32> [[TMP99]], i32 [[TMP96]], i64 3
-; DEFAULT-NEXT: [[TMP101:%.*]] = load i32, ptr [[TMP123]], align 4
-; DEFAULT-NEXT: [[TMP102:%.*]] = load i32, ptr [[TMP125]], align 4
-; DEFAULT-NEXT: [[TMP103:%.*]] = load i32, ptr [[TMP127]], align 4
-; DEFAULT-NEXT: [[TMP104:%.*]] = load i32, ptr [[TMP76]], align 4
-; DEFAULT-NEXT: [[TMP105:%.*]] = insertelement <4 x i32> poison, i32 [[TMP101]], i64 0
-; DEFAULT-NEXT: [[TMP106:%.*]] = insertelement <4 x i32> [[TMP105]], i32 [[TMP102]], i64 1
-; DEFAULT-NEXT: [[TMP107:%.*]] = insertelement <4 x i32> [[TMP106]], i32 [[TMP103]], i64 2
-; DEFAULT-NEXT: [[TMP108:%.*]] = insertelement <4 x i32> [[TMP107]], i32 [[TMP104]], i64 3
+; DEFAULT-NEXT: [[WIDE_GEP12:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP50]]
+; DEFAULT-NEXT: [[WIDE_GEP13:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP51]]
+; DEFAULT-NEXT: [[WIDE_GEP14:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP52]]
+; DEFAULT-NEXT: [[TMP84:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP12]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[TMP92:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP13]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[TMP100:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP14]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; DEFAULT-NEXT: [[TMP108:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP14]], <4 x i1> splat (i1 true), <4 x i32> poison)
; DEFAULT-NEXT: [[TMP109:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI]]
-; DEFAULT-NEXT: [[TMP110:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI4]]
-; DEFAULT-NEXT: [[TMP111:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI5]]
-; DEFAULT-NEXT: [[TMP112:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI6]]
+; DEFAULT-NEXT: [[TMP110:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER9]], [[VEC_PHI4]]
+; DEFAULT-NEXT: [[TMP111:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER10]], [[VEC_PHI5]]
+; DEFAULT-NEXT: [[TMP112:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER11]], [[VEC_PHI6]]
; DEFAULT-NEXT: [[TMP113]] = or <4 x i32> [[TMP109]], [[TMP84]]
; DEFAULT-NEXT: [[TMP114]] = or <4 x i32> [[TMP110]], [[TMP92]]
; DEFAULT-NEXT: [[TMP115]] = or <4 x i32> [[TMP111]], [[TMP100]]
@@ -160,13 +100,13 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT: [[SCALAR_RECUR_INIT11:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: [[SCALAR_RECUR_INIT21:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP118]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: br label %[[LOOP1:.*]]
; DEFAULT: [[LOOP1]]:
-; DEFAULT-NEXT: [[TMP129:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP68:%.*]], %[[LOOP1]] ]
-; DEFAULT-NEXT: [[SCALAR_RECUR15:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT11]], %[[SCALAR_PH]] ], [ [[TMP129]], %[[LOOP1]] ]
+; DEFAULT-NEXT: [[TMP53:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP68:%.*]], %[[LOOP1]] ]
+; DEFAULT-NEXT: [[SCALAR_RECUR15:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT21]], %[[SCALAR_PH]] ], [ [[TMP53]], %[[LOOP1]] ]
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; DEFAULT-NEXT: [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_2:%.*]], %[[LOOP1]] ]
; DEFAULT-NEXT: [[TMP67:%.*]] = add i64 [[Y]], 1
@@ -224,22 +164,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: [[TMP117:%.*]] = and <4 x i32> [[BROADCAST_SPLAT2]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = xor <4 x i32> [[TMP117]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP10:%.*]] = zext <4 x i32> [[TMP9]] to <4 x i64>
-; VSCALEFORTUNING2-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP11]]
-; VSCALEFORTUNING2-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP10]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP13]]
-; VSCALEFORTUNING2-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP10]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP120:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP15]]
-; VSCALEFORTUNING2-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP10]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP17]]
-; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP12]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP14]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP120]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i64 3
+; VSCALEFORTUNING2-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP10]]
; VSCALEFORTUNING2-NEXT: br label %[[VECTOR_BODY:.*]]
; VSCALEFORTUNING2: [[VECTOR_BODY]]:
; VSCALEFORTUNING2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -269,72 +194,27 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: [[TMP44:%.*]] = or <4 x i32> [[TMP41]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP45:%.*]] = or <4 x i32> [[TMP42]], [[BROADCAST_SPLAT]]
; VSCALEFORTUNING2-NEXT: [[TMP46:%.*]] = or <4 x i32> [[TMP43]], [[BROADCAST_SPLAT]]
+; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[WIDE_MASKED_GATHER11:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[TMP47:%.*]] = lshr <4 x i32> [[TMP44]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP48:%.*]] = lshr <4 x i32> [[TMP45]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP121:%.*]] = lshr <4 x i32> [[TMP46]], splat (i32 1)
; VSCALEFORTUNING2-NEXT: [[TMP50:%.*]] = zext <4 x i32> [[TMP47]] to <4 x i64>
; VSCALEFORTUNING2-NEXT: [[TMP51:%.*]] = zext <4 x i32> [[TMP48]] to <4 x i64>
; VSCALEFORTUNING2-NEXT: [[TMP52:%.*]] = zext <4 x i32> [[TMP121]] to <4 x i64>
-; VSCALEFORTUNING2-NEXT: [[TMP53:%.*]] = extractelement <4 x i64> [[TMP50]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP53]]
-; VSCALEFORTUNING2-NEXT: [[TMP122:%.*]] = extractelement <4 x i64> [[TMP50]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP123:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP122]]
-; VSCALEFORTUNING2-NEXT: [[TMP124:%.*]] = extractelement <4 x i64> [[TMP50]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP124]]
-; VSCALEFORTUNING2-NEXT: [[TMP126:%.*]] = extractelement <4 x i64> [[TMP50]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP127:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP126]]
-; VSCALEFORTUNING2-NEXT: [[TMP128:%.*]] = extractelement <4 x i64> [[TMP51]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP129:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP128]]
-; VSCALEFORTUNING2-NEXT: [[TMP130:%.*]] = extractelement <4 x i64> [[TMP51]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP131:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP130]]
-; VSCALEFORTUNING2-NEXT: [[TMP65:%.*]] = extractelement <4 x i64> [[TMP51]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP65]]
-; VSCALEFORTUNING2-NEXT: [[TMP67:%.*]] = extractelement <4 x i64> [[TMP51]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP67]]
-; VSCALEFORTUNING2-NEXT: [[TMP69:%.*]] = extractelement <4 x i64> [[TMP52]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP69]]
-; VSCALEFORTUNING2-NEXT: [[TMP71:%.*]] = extractelement <4 x i64> [[TMP52]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP72:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP71]]
-; VSCALEFORTUNING2-NEXT: [[TMP73:%.*]] = extractelement <4 x i64> [[TMP52]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP74:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP73]]
-; VSCALEFORTUNING2-NEXT: [[TMP75:%.*]] = extractelement <4 x i64> [[TMP52]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr [[SRC_2]], i64 [[TMP75]]
-; VSCALEFORTUNING2-NEXT: [[TMP77:%.*]] = load i32, ptr [[TMP54]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP123]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP79:%.*]] = load i32, ptr [[TMP125]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP127]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP81:%.*]] = insertelement <4 x i32> poison, i32 [[TMP77]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP82:%.*]] = insertelement <4 x i32> [[TMP81]], i32 [[TMP78]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP83:%.*]] = insertelement <4 x i32> [[TMP82]], i32 [[TMP79]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP84:%.*]] = insertelement <4 x i32> [[TMP83]], i32 [[TMP80]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP129]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP86:%.*]] = load i32, ptr [[TMP131]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP87:%.*]] = load i32, ptr [[TMP66]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP68]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP89:%.*]] = insertelement <4 x i32> poison, i32 [[TMP85]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP90:%.*]] = insertelement <4 x i32> [[TMP89]], i32 [[TMP86]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP91:%.*]] = insertelement <4 x i32> [[TMP90]], i32 [[TMP87]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP92:%.*]] = insertelement <4 x i32> [[TMP91]], i32 [[TMP88]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP93:%.*]] = load i32, ptr [[TMP70]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP94:%.*]] = load i32, ptr [[TMP72]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP95:%.*]] = load i32, ptr [[TMP74]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP96:%.*]] = load i32, ptr [[TMP76]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP97:%.*]] = insertelement <4 x i32> poison, i32 [[TMP93]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP98:%.*]] = insertelement <4 x i32> [[TMP97]], i32 [[TMP94]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP99:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP95]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP100:%.*]] = insertelement <4 x i32> [[TMP99]], i32 [[TMP96]], i64 3
-; VSCALEFORTUNING2-NEXT: [[TMP101:%.*]] = load i32, ptr [[TMP70]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP102:%.*]] = load i32, ptr [[TMP72]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP103:%.*]] = load i32, ptr [[TMP74]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP104:%.*]] = load i32, ptr [[TMP76]], align 4
-; VSCALEFORTUNING2-NEXT: [[TMP105:%.*]] = insertelement <4 x i32> poison, i32 [[TMP101]], i64 0
-; VSCALEFORTUNING2-NEXT: [[TMP106:%.*]] = insertelement <4 x i32> [[TMP105]], i32 [[TMP102]], i64 1
-; VSCALEFORTUNING2-NEXT: [[TMP107:%.*]] = insertelement <4 x i32> [[TMP106]], i32 [[TMP103]], i64 2
-; VSCALEFORTUNING2-NEXT: [[TMP108:%.*]] = insertelement <4 x i32> [[TMP107]], i32 [[TMP104]], i64 3
+; VSCALEFORTUNING2-NEXT: [[WIDE_GEP12:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP50]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_GEP13:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP51]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_GEP14:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP52]]
+; VSCALEFORTUNING2-NEXT: [[TMP84:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP12]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[TMP92:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP13]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[TMP100:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP14]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VSCALEFORTUNING2-NEXT: [[TMP108:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP14]], <4 x i1> splat (i1 true), <4 x i32> poison)
; VSCALEFORTUNING2-NEXT: [[TMP109:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI]]
-; VSCALEFORTUNING2-NEXT: [[TMP110:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI4]]
-; VSCALEFORTUNING2-NEXT: [[TMP111:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI5]]
-; VSCALEFORTUNING2-NEXT: [[TMP112:%.*]] = or <4 x i32> [[TMP26]], [[VEC_PHI6]]
+; VSCALEFORTUNING2-NEXT: [[TMP110:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER9]], [[VEC_PHI4]]
+; VSCALEFORTUNING2-NEXT: [[TMP111:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER10]], [[VEC_PHI5]]
+; VSCALEFORTUNING2-NEXT: [[TMP112:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER11]], [[VEC_PHI6]]
; VSCALEFORTUNING2-NEXT: [[TMP113]] = or <4 x i32> [[TMP109]], [[TMP84]]
; VSCALEFORTUNING2-NEXT: [[TMP114]] = or <4 x i32> [[TMP110]], [[TMP92]]
; VSCALEFORTUNING2-NEXT: [[TMP115]] = or <4 x i32> [[TMP111]], [[TMP100]]
@@ -352,13 +232,13 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VSCALEFORTUNING2: [[SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT11:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VSCALEFORTUNING2-NEXT: [[SCALAR_RECUR_INIT21:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP118]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
-; VSCALEFORTUNING2-NEXT: [[TMP132:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP57:%.*]], %[[LOOP]] ]
-; VSCALEFORTUNING2-NEXT: [[TMP55:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT11]], %[[SCALAR_PH]] ], [ [[TMP132]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[TMP53:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP57:%.*]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[TMP55:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT21]], %[[SCALAR_PH]] ], [ [[TMP53]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_2:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[TMP56:%.*]] = add i64 [[Y]], 1
@@ -397,63 +277,53 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[Y]], 1
; PRED-NEXT: br label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
-; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; PRED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; PRED-NEXT: [[TMP11:%.*]] = add i64 [[Y]], 1
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP11]]
; PRED-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP12]], align 4
-; PRED-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP5]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT: [[TMP13:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
-; PRED-NEXT: [[TMP14:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
-; PRED-NEXT: [[TMP15:%.*]] = or <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; PRED-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; PRED-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; PRED-NEXT: [[TMP7:%.*]] = shl <4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; PRED-NEXT: [[TMP6:%.*]] = or <4 x i32> [[TMP4]], [[TMP7]]
; PRED-NEXT: [[TMP20:%.*]] = or i32 [[Z]], [[X]]
-; PRED-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP20]], i64 0
-; PRED-NEXT: [[TMP16:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT: [[TMP17:%.*]] = and <vscale x 4 x i32> [[TMP16]], splat (i32 1)
-; PRED-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i32> [[TMP17]], splat (i32 1)
-; PRED-NEXT: [[TMP19:%.*]] = zext <vscale x 4 x i32> [[TMP18]] to <vscale x 4 x i64>
-; PRED-NEXT: [[DOTSPLAT:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP19]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 4
-; PRED-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1
-; PRED-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP24]]
-; PRED-NEXT: [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT: [[TMP26:%.*]] = mul nuw i32 [[TMP25]], 4
-; PRED-NEXT: [[TMP27:%.*]] = sub i32 [[TMP26]], 1
-; PRED-NEXT: [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP27]]
+; PRED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[TMP20]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; PRED-NEXT: [[TMP8:%.*]] = and <4 x i32> [[BROADCAST_SPLAT2]], splat (i32 1)
+; PRED-NEXT: [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], splat (i32 1)
+; PRED-NEXT: [[TMP10:%.*]] = zext <4 x i32> [[TMP9]] to <4 x i64>
+; PRED-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP10]]
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
-; PRED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT6]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[VECTOR_RECUR4:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT3]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[TMP29]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[BROADCAST_SPLAT6]], i32 1)
-; PRED-NEXT: [[TMP30:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR4]], <vscale x 4 x i32> [[TMP29]], i32 1)
-; PRED-NEXT: [[TMP31:%.*]] = or <vscale x 4 x i32> [[TMP30]], [[BROADCAST_SPLAT]]
-; PRED-NEXT: [[TMP32:%.*]] = shl <vscale x 4 x i32> [[TMP31]], splat (i32 1)
-; PRED-NEXT: [[TMP33:%.*]] = or <vscale x 4 x i32> [[TMP32]], splat (i32 2)
-; PRED-NEXT: [[TMP34:%.*]] = or <vscale x 4 x i32> [[TMP15]], [[TMP33]]
-; PRED-NEXT: [[TMP35:%.*]] = or <vscale x 4 x i32> [[TMP34]], [[BROADCAST_SPLAT]]
-; PRED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[DOTSPLAT]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; PRED-NEXT: [[TMP36:%.*]] = lshr <vscale x 4 x i32> [[TMP35]], splat (i32 1)
-; PRED-NEXT: [[TMP37:%.*]] = zext <vscale x 4 x i32> [[TMP36]] to <vscale x 4 x i64>
-; PRED-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP37]]
-; PRED-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP38]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; PRED-NEXT: [[TMP39:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
-; PRED-NEXT: [[TMP40:%.*]] = or <vscale x 4 x i32> [[TMP39]], [[WIDE_MASKED_GATHER7]]
-; PRED-NEXT: [[TMP41]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP40]], <vscale x 4 x i32> [[VEC_PHI]]
-; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], [[TMP2]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP43:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT5]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[VECTOR_RECUR3:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[TMP23]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[BROADCAST_SPLAT5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; PRED-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR3]], <4 x i32> [[TMP23]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; PRED-NEXT: [[TMP13:%.*]] = or <4 x i32> [[TMP24]], [[BROADCAST_SPLAT]]
+; PRED-NEXT: [[TMP14:%.*]] = shl <4 x i32> [[TMP13]], splat (i32 1)
+; PRED-NEXT: [[TMP15:%.*]] = or <4 x i32> [[TMP14]], splat (i32 2)
+; PRED-NEXT: [[TMP16:%.*]] = or <4 x i32> [[TMP6]], [[TMP15]]
+; PRED-NEXT: [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[BROADCAST_SPLAT]]
+; PRED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; PRED-NEXT: [[TMP18:%.*]] = lshr <4 x i32> [[TMP17]], splat (i32 1)
+; PRED-NEXT: [[TMP19:%.*]] = zext <4 x i32> [[TMP18]] to <4 x i64>
+; PRED-NEXT: [[WIDE_GEP6:%.*]] = getelementptr i32, ptr [[SRC_2]], <4 x i64> [[TMP19]]
+; PRED-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP6]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; PRED-NEXT: [[TMP25:%.*]] = or <4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
+; PRED-NEXT: [[TMP21:%.*]] = or <4 x i32> [[TMP25]], [[WIDE_MASKED_GATHER7]]
+; PRED-NEXT: [[TMP22]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP21]], <4 x i32> [[VEC_PHI]]
+; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; PRED-NEXT: [[TMP43:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; PRED-NEXT: [[TMP42:%.*]] = xor i1 [[TMP43]], true
; PRED-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
-; PRED-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP41]])
+; PRED-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP22]])
; PRED-NEXT: br label %[[EXIT:.*]]
; PRED: [[EXIT]]:
; PRED-NEXT: ret i32 [[TMP44]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index f14e913c9e15a..3c1854e7fff25 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -46,12 +46,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x i32>, ptr [[TMP18]], align 4
; CHECK-NEXT: [[TMP16:%.*]] = zext <2 x i32> [[WIDE_LOAD5]] to <2 x i64>
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i64> [[TMP16]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP17]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP16]], i64 1
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP9]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x ptr> [[TMP20]], ptr [[TMP19]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <2 x i64> [[TMP16]]
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.v2p0.i32(<2 x ptr> [[TMP13]], i32 1, <2 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
index c7b1bcdf31e45..a615928c6a3f7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -516,29 +516,34 @@ exit:
define void @interleave_group_with_gather(ptr %indices, ptr %src, i64 %n) {
; CHECK-LABEL: define void @interleave_group_with_gather(
; CHECK-SAME: ptr [[INDICES:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[OUT_GEP:%.*]] = getelementptr { double, double }, ptr null, i64 [[IV]]
; CHECK-NEXT: [[IDX_GEP:%.*]] = getelementptr i32, ptr [[INDICES]], i64 [[IV]]
-; CHECK-NEXT: [[IDX:%.*]] = load i32, ptr [[IDX_GEP]], align 4
-; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[IDX]] to i64
-; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[SRC_VAL:%.*]] = load double, ptr [[SRC_GEP]], align 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[IDX_GEP]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr double, ptr [[SRC]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x double> poison)
; CHECK-NEXT: [[OUT_M1_0_GEP:%.*]] = getelementptr i8, ptr [[OUT_GEP]], i64 -16
-; CHECK-NEXT: [[OUT_M1_0:%.*]] = load double, ptr [[OUT_M1_0_GEP]], align 8
-; CHECK-NEXT: [[ADD_0:%.*]] = fadd double 1.000000e+01, [[SRC_VAL]]
-; CHECK-NEXT: store double [[ADD_0]], ptr [[OUT_M1_0_GEP]], align 8
-; CHECK-NEXT: [[OUT_M1_1_GEP:%.*]] = getelementptr i8, ptr [[OUT_GEP]], i64 -8
-; CHECK-NEXT: [[OUT_M1_1:%.*]] = load double, ptr [[OUT_M1_1_GEP]], align 8
-; CHECK-NEXT: [[ADD_1:%.*]] = fadd double 1.000000e+01, [[SRC_VAL]]
-; CHECK-NEXT: store double [[ADD_1]], ptr [[OUT_M1_1_GEP]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[OUT_M1_0_GEP]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
index 18712ad5d1fba..b147170c292f5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
@@ -9,28 +9,26 @@ define i32 @gep_with_all_invariant_operands(ptr %src.0, ptr %src.1, i64 %n, i1 %
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[N]]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[COND]], ptr [[SRC_1]], ptr [[TMP8]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[BROADCAST_SPLAT]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP12:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[WIDE_MASKED_GATHER]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP16]]
>From 0c72014fb980c58762c2365c258ace629836f7ca Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 12 Aug 2026 18:14:21 +0100
Subject: [PATCH 2/3] [LV] Factor costInterleaveGatherScatter (NFC)
The motivation for factoring out a costInterleaveGatherScatter that
compares the cost of interleaving versus that of a gather-scatter is for
re-use in a follow-up doing VPlan-based gather-scatter-widening.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 14 +-
.../Vectorize/LoopVectorizationPlanner.h | 7 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 127 +++++++++---------
3 files changed, 70 insertions(+), 78 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index dc4b640eff109..64136846a80bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -146,17 +146,11 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
: TTI.isLegalMaskedStore(ScalarTy, Alignment, AddressSpace));
}
-bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
- ElementCount VF) const {
- bool LI = isa<LoadInst>(V);
- bool SI = isa<StoreInst>(V);
- if (!LI && !SI)
- return false;
- auto *Ty = getLoadStoreType(V);
- Align Align = getLoadStoreAlignment(V);
+bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const {
return ForceTargetSupportsGatherScatterOps ||
- (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
- (SI && TTI.isLegalMaskedScatter(Ty, Align));
+ (IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
+ : TTI.isLegalMaskedScatter(ScalarTy, Alignment));
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..68eebc8a1c3ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -787,9 +787,10 @@ class VFSelectionContext {
bool isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy, Align Alignment,
unsigned AddressSpace) const;
- /// Returns true if the target machine can represent \p V as a masked gather
- /// or scatter operation.
- bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
+ /// Returns true if the target machine supports a gather (if \p IsLoad)
+ /// or scatter of scalar type \p ScalarTy with \p Alignment.
+ bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const;
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d892ff2a0fa2b..c7b9a815e420e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1034,11 +1034,11 @@ class LoopVectorizationCostModel {
/// every iteration of the loop header.
inline uint64_t
getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
- const BasicBlock *BB);
+ const BasicBlock *BB) const;
/// Returns true if an artificially high cost for emulated masked memrefs
/// should be used.
- bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
+ bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
/// Return the costs for our two available strategies for lowering a
/// div/rem operation which requires speculating at least one lane.
@@ -1063,6 +1063,10 @@ class LoopVectorizationCostModel {
/// consecutive or part of an interleave group.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
+ /// Returns true if the target machine supports gather or scatter for \p I's
+ /// data type and alignment.
+ bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
+
/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) const {
return InterleaveInfo.isInterleaved(Instr);
@@ -1348,17 +1352,47 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ /// Pick between interleave and gather-scatter based on cost. Returns a pair
+ /// of widening decision along with corresponding cost.
+ std::pair<InstWidening, InstructionCost>
+ costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ InstructionCost InterleaveCost = InstructionCost::getInvalid();
+ unsigned NumAccesses = 1;
+ if (isAccessInterleaved(I)) {
+ const auto *Group = getInterleavedAccessGroup(I);
+ assert(Group && "Fail to get an interleaved access group.");
+
+ if (interleavedAccessCanBeWidened(I, VF)) {
+ NumAccesses = Group->getNumMembers();
+ InterleaveCost = getInterleaveGroupCost(I, VF);
+ }
+ }
+ InstructionCost GatherScatterCost =
+ isLegalGatherOrScatter(I, VF)
+ ? getGatherScatterCost(I, VF) * NumAccesses
+ : InstructionCost::getInvalid();
+ InstructionCost ScalarizationCost =
+ getMemInstScalarizationCost(I, VF) * NumAccesses;
+ if (InterleaveCost <= GatherScatterCost &&
+ InterleaveCost < ScalarizationCost)
+ return {CM_Interleave, InterleaveCost};
+ if (GatherScatterCost < ScalarizationCost)
+ return {CM_GatherScatter, GatherScatterCost};
+ return {CM_Scalarize, ScalarizationCost};
+ }
+
/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
/// The cost computation for scalarized memory instruction.
- InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
+ InstructionCost getMemInstScalarizationCost(Instruction *I,
+ ElementCount VF) const;
/// The cost computation for interleaving group of memory instructions.
- InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
+ InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
/// The cost computation for Gather/Scatter instruction.
- InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
+ InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
/// The cost computation for widening instruction \p I with consecutive
/// memory access.
@@ -1517,15 +1551,6 @@ class LoopVectorizationCostModel {
/// unless necessary, e.g. when the loop isn't legal to vectorize or when
/// there is no predication.
std::function<BlockFrequencyInfo &()> GetBFI;
- /// The BlockFrequencyInfo returned from GetBFI.
- BlockFrequencyInfo *BFI = nullptr;
- /// Returns the BlockFrequencyInfo for the function if cached, otherwise
- /// fetches it via GetBFI. Avoids an indirect call to the std::function.
- BlockFrequencyInfo &getBFI() {
- if (!BFI)
- BFI = &GetBFI();
- return *BFI;
- }
const Function *TheFunction;
@@ -2400,6 +2425,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
getLoadStoreAddressSpace(I));
}
+bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
+ ElementCount VF) const {
+ assert((isa<LoadInst, StoreInst>(I)));
+ return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
+ getLoadStoreAlignment(I));
+}
+
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
ElementCount VF) {
if (!isPredicatedInst(I))
@@ -2423,7 +2455,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I),
getLoadStorePointerOperand(I));
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
- !Config.isLegalGatherOrScatter(I, VF);
+ !isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
case Instruction::SDiv:
@@ -2500,7 +2532,7 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
}
uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
- TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
+ TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) const {
if (CostKind == TTI::TCK_CodeSize)
return 1;
// If the block wasn't originally predicated then return early to avoid
@@ -2509,8 +2541,8 @@ uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
return 1;
uint64_t HeaderFreq =
- getBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
- uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
+ GetBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
+ uint64_t BBFreq = GetBFI().getBlockFreq(BB).getFrequency();
assert(HeaderFreq >= BBFreq &&
"Header has smaller block freq than dominated BB?");
return std::round((double)HeaderFreq / BBFreq);
@@ -2583,8 +2615,6 @@ LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
Instruction *I, ElementCount VF) const {
assert(isAccessInterleaved(I) && "Expecting interleaved access.");
- assert(getWideningDecision(I, VF) == CM_Unknown &&
- "Decision should not be set yet.");
auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Must have a group.");
unsigned InterleaveFactor = Group->getFactor();
@@ -3989,8 +4019,8 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return 1;
}
-bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
- ElementCount VF) {
+bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
+ Instruction *I, ElementCount VF) const {
// TODO: Cost model for emulated masked load/store is completely
// broken. This hack guides the cost model to use an artificially
// high enough value to practically disable vectorization with such
@@ -4226,7 +4256,7 @@ static const SCEV *getAddressAccessSCEV(
InstructionCost
LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(VF.isVector() &&
"Scalarization cost of instruction implies vectorization.");
if (VF.isScalable())
@@ -4351,7 +4381,7 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
const Align Alignment = getLoadStoreAlignment(I);
@@ -4374,7 +4404,7 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -4721,9 +4751,8 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
};
const InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
+ isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
+ : InstructionCost::getInvalid();
// Load: Scalar load + broadcast
// Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
@@ -4751,45 +4780,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
continue;
}
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- InstructionCost InterleaveCost = InstructionCost::getInvalid();
- unsigned NumAccesses = 1;
- if (isAccessInterleaved(&I)) {
- const auto *Group = getInterleavedAccessGroup(&I);
- assert(Group && "Fail to get an interleaved access group.");
-
- // Make one decision for the whole group.
- if (getWideningDecision(&I, VF) != CM_Unknown)
- continue;
-
- NumAccesses = Group->getNumMembers();
- if (interleavedAccessCanBeWidened(&I, VF))
- InterleaveCost = getInterleaveGroupCost(&I, VF);
- }
-
- InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF) * NumAccesses
- : InstructionCost::getInvalid();
+ // Make one decision for the whole interleave group.
+ if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
+ continue;
- InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(&I, VF) * NumAccesses;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
- // Choose better solution for the current VF,
- // write down this decision and use it during vectorization.
- InstructionCost Cost;
- InstWidening Decision;
- if (InterleaveCost <= GatherScatterCost &&
- InterleaveCost < ScalarizationCost) {
- Decision = CM_Interleave;
- Cost = InterleaveCost;
- } else if (GatherScatterCost < ScalarizationCost) {
- Decision = CM_GatherScatter;
- Cost = GatherScatterCost;
- } else {
- Decision = CM_Scalarize;
- Cost = ScalarizationCost;
- }
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
>From 16ef5bebb2f34199a78166847a8ad25514abda1d Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 07:38:24 +0100
Subject: [PATCH 3/3] [LV] Absorb uniform case as well
---
.../Transforms/Vectorize/LoopVectorize.cpp | 86 +++++++++----------
1 file changed, 39 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c7b9a815e420e..0f922bed39349 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1352,13 +1352,37 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ bool isLegalToScalarize(Instruction *I, ElementCount VF) const {
+ if (!VF.isScalable())
+ // Scalarization of fixed length vectors "just works".
+ return true;
+
+ // We have dedicated lowering for unpredicated uniform loads and
+ // stores. Note that even with tail folding we know that at least
+ // one lane is active (i.e. generalized predication is not possible
+ // here), and the logic below depends on this fact.
+ if (!foldTailByMasking())
+ return true;
+
+ // For scalable vectors, a uniform memop load is always
+ // uniform-by-parts and we know how to scalarize that.
+ if (isa<LoadInst>(I))
+ return true;
+
+ // A uniform store isn't neccessarily uniform-by-part
+ // and we can't assume scalarization.
+ auto *SI = cast<StoreInst>(I);
+ return TheLoop->isLoopInvariant(SI->getValueOperand());
+ };
+
/// Pick between interleave and gather-scatter based on cost. Returns a pair
/// of widening decision along with corresponding cost.
std::pair<InstWidening, InstructionCost>
costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ bool IsUniform = isUniformMemOp(*I, VF);
InstructionCost InterleaveCost = InstructionCost::getInvalid();
unsigned NumAccesses = 1;
- if (isAccessInterleaved(I)) {
+ if (!IsUniform && isAccessInterleaved(I)) {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -1367,13 +1391,20 @@ class LoopVectorizationCostModel {
InterleaveCost = getInterleaveGroupCost(I, VF);
}
}
+
InstructionCost GatherScatterCost =
isLegalGatherOrScatter(I, VF)
? getGatherScatterCost(I, VF) * NumAccesses
: InstructionCost::getInvalid();
+
+ // FIXME: This cost is a significant under-estimate for tail folded
+ // memory ops.
InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(I, VF) * NumAccesses;
- if (InterleaveCost <= GatherScatterCost &&
+ IsUniform ? (isLegalToScalarize(I, VF) ? getUniformMemOpCost(I, VF)
+ : InstructionCost::getInvalid())
+ : getMemInstScalarizationCost(I, VF) * NumAccesses;
+
+ if (!IsUniform && InterleaveCost <= GatherScatterCost &&
InterleaveCost < ScalarizationCost)
return {CM_Interleave, InterleaveCost};
if (GatherScatterCost < ScalarizationCost)
@@ -1403,7 +1434,7 @@ class LoopVectorizationCostModel {
/// Load: scalar load + broadcast.
/// Store: scalar store + (loop invariant value stored? 0 : extract of last
/// element)
- InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
+ InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
/// Estimate the overhead of scalarizing an instruction. This is a
/// convenience wrapper for the type-based getScalarizationOverhead API.
@@ -4346,7 +4377,7 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
InstructionCost
LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(isUniformMemOp(*I, VF));
Type *ValTy = getLoadStoreType(I);
@@ -4726,49 +4757,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (!Ptr)
continue;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
if (isUniformMemOp(I, VF)) {
- auto IsLegalToScalarize = [&]() {
- if (!VF.isScalable())
- // Scalarization of fixed length vectors "just works".
- return true;
-
- // We have dedicated lowering for unpredicated uniform loads and
- // stores. Note that even with tail folding we know that at least
- // one lane is active (i.e. generalized predication is not possible
- // here), and the logic below depends on this fact.
- if (!foldTailByMasking())
- return true;
-
- // For scalable vectors, a uniform memop load is always
- // uniform-by-parts and we know how to scalarize that.
- if (isa<LoadInst>(I))
- return true;
-
- // A uniform store isn't neccessarily uniform-by-part
- // and we can't assume scalarization.
- auto &SI = cast<StoreInst>(I);
- return TheLoop->isLoopInvariant(SI.getValueOperand());
- };
-
- const InstructionCost GatherScatterCost =
- isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
-
- // Load: Scalar load + broadcast
- // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
- // FIXME: This cost is a significant under-estimate for tail folded
- // memory ops.
- const InstructionCost ScalarizationCost =
- IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
- : InstructionCost::getInvalid();
-
// Choose better solution for the current VF, Note that Invalid
// costs compare as maximumal large. If both are invalid, we get
// scalable invalid which signals a failure and a vectorization abort.
- if (GatherScatterCost < ScalarizationCost)
- setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
- else
- setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
+ setWideningDecision(&I, VF, Decision, Cost);
continue;
}
@@ -4784,9 +4779,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
continue;
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
-
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
More information about the llvm-commits
mailing list