[llvm] 308393e - [LV] Allow scalable epilogue VFs matching the MainLoop VF (#208686)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 05:30:41 PDT 2026
Author: Benjamin Maxwell
Date: 2026-07-13T13:30:37+01:00
New Revision: 308393ede14cc8533d1be122b5df88bf9ba4173b
URL: https://github.com/llvm/llvm-project/commit/308393ede14cc8533d1be122b5df88bf9ba4173b
DIFF: https://github.com/llvm/llvm-project/commit/308393ede14cc8533d1be122b5df88bf9ba4173b.diff
LOG: [LV] Allow scalable epilogue VFs matching the MainLoop VF (#208686)
This effectively limited scalable epilogue VFs to the MainLoopVF/2, as
all VFs are powers-of-two. This restriction does not exist for fixed
vector VFs. This could result in worse VF selections for scalable
epilogues (at high interleave counts), or in many cases only fixed
vectors selected (as there would be no allowed scalable VF in
ProfitableVFs).
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll
llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a4aae2651ee4c..1929f52ae7d95 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3592,14 +3592,11 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
VPlan &CurrentPlan = getPlanFor(NextVF.Width);
ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
- // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
- // vectors) or > the VF of the main loop (fixed vectors).
+ // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
+ // the VF of the main loop.
if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
- ElementCount::isKnownGE(EffectiveVF, EstimatedRuntimeVF)) ||
- (EffectiveVF.isScalable() &&
- ElementCount::isKnownGE(EffectiveVF, MainLoopVF)) ||
- (!EffectiveVF.isScalable() && !MainLoopVF.isScalable() &&
- ElementCount::isKnownGT(EffectiveVF, MainLoopVF)))
+ ElementCount::isKnownGT(EffectiveVF, EstimatedRuntimeVF)) ||
+ ElementCount::isKnownGT(EffectiveVF, MainLoopVF))
continue;
// If EffectiveVF is greater than the number of remaining iterations, the
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 5b3c73078b2b0..33c781b99a9bc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -13,10 +13,10 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 2
+; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -72,24 +72,24 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; DEFAULT-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 2
+; DEFAULT-NEXT: [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 3
; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP34]]
; DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP35:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT8]] to <vscale x 4 x i16>
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP40:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX9]]
-; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP36]], align 1
-; DEFAULT-NEXT: [[TMP37:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD10]] to <vscale x 4 x i16>
-; DEFAULT-NEXT: [[TMP38:%.*]] = mul <vscale x 4 x i16> [[TMP37]], [[TMP35]]
-; DEFAULT-NEXT: [[TMP39:%.*]] = or <vscale x 4 x i16> [[TMP38]], [[TMP37]]
-; DEFAULT-NEXT: [[TMP40:%.*]] = lshr <vscale x 4 x i16> [[TMP39]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP41:%.*]] = trunc <vscale x 4 x i16> [[TMP40]] to <vscale x 4 x i8>
+; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 8 x i8>, ptr [[TMP36]], align 1
+; DEFAULT-NEXT: [[TMP32:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP37:%.*]] = mul <vscale x 8 x i16> [[TMP32]], [[TMP40]]
+; DEFAULT-NEXT: [[TMP38:%.*]] = or <vscale x 8 x i16> [[TMP37]], [[TMP32]]
+; DEFAULT-NEXT: [[TMP35:%.*]] = lshr <vscale x 8 x i16> [[TMP38]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP39:%.*]] = trunc <vscale x 8 x i16> [[TMP35]] to <vscale x 8 x i8>
; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX9]]
-; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP41]], ptr [[TMP42]], align 1
+; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP39]], ptr [[TMP42]], align 1
; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP34]]
; DEFAULT-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC6]]
; DEFAULT-NEXT: br i1 [[TMP43]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 94dde2b58a0a0..d64504685fa5d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -372,7 +372,7 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-LABEL: define i32 @load_factor_4_with_gap(
; CHECK-NOTF-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
-; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
@@ -449,31 +449,31 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10:![0-9]+]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP44]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 4
; CHECK-NOTF-NEXT: [[N_VEC14:%.*]] = sub i64 [[N]], [[N_MOD_VF13]]
-; CHECK-NOTF-NEXT: [[TMP45:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NOTF-NEXT: [[TMP45:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT21:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <2 x i32> [ [[TMP45]], %[[SCALAR_PH]] ], [ [[TMP49:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP45]], %[[SCALAR_PH]] ], [ [[TMP49:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
-; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 3, i32 7>
-; CHECK-NOTF-NEXT: [[TMP47:%.*]] = add <2 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
-; CHECK-NOTF-NEXT: [[TMP48:%.*]] = add <2 x i32> [[TMP47]], [[STRIDED_VEC19]]
-; CHECK-NOTF-NEXT: [[TMP49]] = add <2 x i32> [[TMP48]], [[STRIDED_VEC20]]
-; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 2
+; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = add <4 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = add <4 x i32> [[TMP47]], [[STRIDED_VEC19]]
+; CHECK-NOTF-NEXT: [[TMP49]] = add <4 x i32> [[TMP48]], [[STRIDED_VEC20]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 4
; CHECK-NOTF-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT21]], [[N_VEC14]]
; CHECK-NOTF-NEXT: br i1 [[TMP50]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP49]])
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP49]])
; CHECK-NOTF-NEXT: [[CMP_N22:%.*]] = icmp eq i64 [[N]], [[N_VEC14]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N22]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
@@ -681,7 +681,7 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-LABEL: define i32 @load_factor_4_with_tail_gap(
; CHECK-NOTF-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]]) #[[ATTR2]] {
; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
-; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
@@ -759,33 +759,33 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP46:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX12]])
; CHECK-NOTF-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], 2
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP46]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF13:%.*]] = urem i64 [[N]], 4
; CHECK-NOTF-NEXT: [[TMP47:%.*]] = icmp eq i64 [[N_MOD_VF13]], 0
-; CHECK-NOTF-NEXT: [[TMP48:%.*]] = select i1 [[TMP47]], i64 2, i64 [[N_MOD_VF13]]
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = select i1 [[TMP47]], i64 4, i64 [[N_MOD_VF13]]
; CHECK-NOTF-NEXT: [[N_VEC14:%.*]] = sub i64 [[N]], [[TMP48]]
-; CHECK-NOTF-NEXT: [[TMP49:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NOTF-NEXT: [[TMP49:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT21:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <2 x i32> [ [[TMP49]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP49]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
-; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <2 x i32> <i32 2, i32 6>
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <2 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
-; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <2 x i32> [[TMP51]], [[STRIDED_VEC19]]
-; CHECK-NOTF-NEXT: [[TMP53]] = add <2 x i32> [[TMP52]], [[STRIDED_VEC20]]
-; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 2
+; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <4 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <4 x i32> [[TMP51]], [[STRIDED_VEC19]]
+; CHECK-NOTF-NEXT: [[TMP53]] = add <4 x i32> [[TMP52]], [[STRIDED_VEC20]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT21]] = add nuw i64 [[IV]], 4
; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT21]], [[N_VEC14]]
; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP53]])
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP53]])
; CHECK-NOTF-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
@@ -986,10 +986,13 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br i1 [[CMP]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-NOTF: [[LOOP_PREHEADER]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP47]]
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = shl nuw i64 [[TMP48]], 2
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1075,46 +1078,51 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP51]]
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP50:%.*]] = shl nuw i64 [[TMP49]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], [[TMP50]]
; CHECK-NOTF-NEXT: [[N_VEC25:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF24]]
; CHECK-NOTF-NEXT: [[TMP46:%.*]] = sub i64 [[N]], [[N_VEC25]]
-; CHECK-NOTF-NEXT: [[TMP47:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
; CHECK-NOTF-NEXT: [[INDEX26:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT35:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <2 x i32> [ [[TMP47]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <vscale x 4 x i32> [ [[TMP52]], %[[SCALAR_PH]] ], [ [[TMP63:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[IV:%.*]] = sub i64 [[N]], [[INDEX26]]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV]]
-; CHECK-NOTF-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 -4
-; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <8 x i32>, ptr [[TMP50]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 0, i32 4>
-; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC29]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 1, i32 5>
-; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC31]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <8 x i32> [[WIDE_VEC28]], <8 x i32> poison, <2 x i32> <i32 3, i32 7>
-; CHECK-NOTF-NEXT: [[REVERSE34:%.*]] = shufflevector <2 x i32> [[STRIDED_VEC33]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <2 x i32> [[REVERSE30]], [[VEC_PHI27]]
-; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <2 x i32> [[TMP51]], [[REVERSE32]]
-; CHECK-NOTF-NEXT: [[TMP53]] = add <2 x i32> [[TMP52]], [[REVERSE34]]
-; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], 2
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = sub nuw nsw i64 [[TMP50]], 1
+; CHECK-NOTF-NEXT: [[TMP56:%.*]] = mul i64 [[TMP55]], -4
+; CHECK-NOTF-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 [[TMP56]]
+; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <vscale x 16 x i32>, ptr [[TMP57]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC28]])
+; CHECK-NOTF-NEXT: [[TMP58:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 0
+; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP58]])
+; CHECK-NOTF-NEXT: [[TMP59:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 1
+; CHECK-NOTF-NEXT: [[REVERSE31:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP59]])
+; CHECK-NOTF-NEXT: [[TMP60:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 3
+; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP60]])
+; CHECK-NOTF-NEXT: [[TMP61:%.*]] = add <vscale x 4 x i32> [[REVERSE30]], [[VEC_PHI27]]
+; CHECK-NOTF-NEXT: [[TMP62:%.*]] = add <vscale x 4 x i32> [[TMP61]], [[REVERSE31]]
+; CHECK-NOTF-NEXT: [[TMP63]] = add <vscale x 4 x i32> [[TMP62]], [[REVERSE32]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], [[TMP50]]
; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT35]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP53]])
+; CHECK-NOTF-NEXT: [[TMP65:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP63]])
; CHECK-NOTF-NEXT: [[CMP_N36:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N36]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL37:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL35:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX36:%.*]] = phi i32 [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
; CHECK-NOTF: [[LOOP1]]:
-; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL37]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX38]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL35]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX36]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV1]]
; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[LD1]], [[RDX]]
@@ -1128,7 +1136,7 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[IV1]], 0
; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP1]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: [[RDX_FINAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll b/llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll
index 33b9b1dfff138..141faa96ef830 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll
@@ -55,11 +55,7 @@ define i32 @sext_of_non_memory_op(ptr %src, i32 %offset, i64 %n) #0 {
; CHECK-SAME: ptr [[SRC:%.*]], i32 [[OFFSET:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP43:%.*]] = shl nuw i64 [[TMP9]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[N]] to i32
@@ -69,6 +65,7 @@ define i32 @sext_of_non_memory_op(ptr %src, i32 %offset, i64 %n) #0 {
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP1]], 6
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP8]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -116,37 +113,35 @@ define i32 @sext_of_non_memory_op(ptr %src, i32 %offset, i64 %n) #0 {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP43]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP32]], 3
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP33]]
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], 16
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
-; CHECK-NEXT: [[TMP34:%.*]] = insertelement <vscale x 8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <16 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <vscale x 8 x i32> [ [[TMP34]], %[[VEC_EPILOG_PH]] ], [ [[TMP40:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <16 x i32> [ [[TMP32]], %[[VEC_EPILOG_PH]] ], [ [[TMP39:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP35:%.*]] = trunc i64 [[INDEX12]] to i32
; CHECK-NEXT: [[TMP36:%.*]] = add i32 [[OFFSET]], [[TMP35]]
; CHECK-NEXT: [[TMP37:%.*]] = sext i32 [[TMP36]] to i64
; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP37]]
-; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <vscale x 8 x i8>, ptr [[TMP38]], align 1
-; CHECK-NEXT: [[TMP39:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD14]] to <vscale x 8 x i32>
-; CHECK-NEXT: [[TMP40]] = or <vscale x 8 x i32> [[VEC_PHI13]], [[TMP39]]
-; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX12]], [[TMP33]]
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP38]], align 1
+; CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32>
+; CHECK-NEXT: [[TMP39]] = or <16 x i32> [[VEC_PHI13]], [[TMP34]]
+; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX12]], 16
; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[TMP41]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.or.nxv8i32(<vscale x 8 x i32> [[TMP40]])
+; CHECK-NEXT: [[TMP40:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP39]])
; CHECK-NEXT: [[CMP_N16:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[CMP_N16]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP42]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP40]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -162,7 +157,7 @@ define i32 @sext_of_non_memory_op(ptr %src, i32 %offset, i64 %n) #0 {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP42]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP40]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[OR_LCSSA]]
;
entry:
@@ -192,7 +187,7 @@ attributes #0 = { "target-cpu"="neoverse-512tvb" }
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; CHECK: [[PROF4]] = !{!"branch_weights", i32 8, i32 56}
+; CHECK: [[PROF4]] = !{!"branch_weights", i32 16, i32 48}
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index dd471a175af86..33f6ce0766e96 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -503,10 +503,10 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 1
+; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -561,25 +561,25 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; DEFAULT-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; DEFAULT-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
+; DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 2
; DEFAULT-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP25]]
; DEFAULT-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
-; DEFAULT-NEXT: [[TMP26:%.*]] = insertelement <vscale x 2 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 2 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP29:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP29:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 2 x i16>, ptr [[GEP]], align 2
-; DEFAULT-NEXT: [[TMP28:%.*]] = udiv <vscale x 2 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
-; DEFAULT-NEXT: [[TMP29]] = or <vscale x 2 x i16> [[TMP28]], [[VEC_PHI15]]
+; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
+; DEFAULT-NEXT: [[TMP28:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
+; DEFAULT-NEXT: [[TMP29]] = or <vscale x 4 x i16> [[TMP28]], [[VEC_PHI15]]
; DEFAULT-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[IV]], [[TMP25]]
; DEFAULT-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC11]]
; DEFAULT-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[TMP31:%.*]] = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> [[TMP29]])
+; DEFAULT-NEXT: [[TMP31:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP29]])
; DEFAULT-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; DEFAULT-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
@@ -605,10 +605,10 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[ITER_CHECK:.*]]:
; VSCALEFORTUNING2-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -663,30 +663,30 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 1
+; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP20]]
; VSCALEFORTUNING2-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
-; VSCALEFORTUNING2-NEXT: [[TMP21:%.*]] = insertelement <vscale x 2 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
; VSCALEFORTUNING2-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_VECTOR_BODY]]:
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; VSCALEFORTUNING2-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 2 x i16> [ [[TMP21]], %[[VEC_EPILOG_PH]] ], [ [[TMP24:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 2 x i16>, ptr [[GEP]], align 2
-; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = udiv <vscale x 2 x i16> [[WIDE_LOAD10]], [[BROADCAST_SPLAT7]]
-; VSCALEFORTUNING2-NEXT: [[TMP24]] = or <vscale x 2 x i16> [[TMP23]], [[VEC_PHI9]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
+; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD10]], [[BROADCAST_SPLAT7]]
+; VSCALEFORTUNING2-NEXT: [[TMP21]] = or <vscale x 4 x i16> [[TMP24]], [[VEC_PHI9]]
; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP20]]
; VSCALEFORTUNING2-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = call i16 @llvm.vector.reduce.or.nxv2i16(<vscale x 2 x i16> [[TMP24]])
+; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP21]])
; VSCALEFORTUNING2-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; VSCALEFORTUNING2: [[VEC_EPILOG_SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
; VSCALEFORTUNING2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -699,7 +699,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; PRED-LABEL: define i16 @reduce_udiv(
@@ -708,8 +708,8 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; PRED-NEXT: br label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
-; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
+; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
@@ -717,22 +717,22 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[TMP14:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> poison)
-; PRED-NEXT: [[TMP20:%.*]] = udiv <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
-; PRED-NEXT: [[TMP21:%.*]] = or <vscale x 8 x i16> [[TMP20]], [[VEC_PHI]]
-; PRED-NEXT: [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> [[TMP21]], <vscale x 8 x i16> [[VEC_PHI]]
-; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; PRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP3]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> poison)
+; PRED-NEXT: [[TMP4:%.*]] = udiv <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; PRED-NEXT: [[TMP5:%.*]] = or <vscale x 8 x i16> [[TMP4]], [[VEC_PHI]]
+; PRED-NEXT: [[TMP6]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[VEC_PHI]]
+; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP15:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PRED-NEXT: [[TMP17:%.*]] = xor i1 [[TMP15]], true
-; PRED-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; PRED-NEXT: [[TMP7:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[TMP8:%.*]] = xor i1 [[TMP7]], true
+; PRED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
-; PRED-NEXT: [[TMP19:%.*]] = call i16 @llvm.vector.reduce.or.nxv8i16(<vscale x 8 x i16> [[TMP16]])
+; PRED-NEXT: [[TMP9:%.*]] = call i16 @llvm.vector.reduce.or.nxv8i16(<vscale x 8 x i16> [[TMP6]])
; PRED-NEXT: br label %[[EXIT:.*]]
; PRED: [[EXIT]]:
-; PRED-NEXT: ret i16 [[TMP19]]
+; PRED-NEXT: ret i16 [[TMP9]]
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
index e8010f6581e82..f0dd16ab571f3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
@@ -189,7 +189,7 @@ exit:
; DEBUG: LV: Checking a loop in 'main_vf_vscale_x_2'
; DEBUG: Create Skeleton for epilogue vectorized loop (first pass)
-; DEBUG: Main Loop VF:vscale x 2, Main Loop UF:2, Epilogue Loop VF:8, Epilogue Loop UF:1
+; DEBUG: Main Loop VF:vscale x 2, Main Loop UF:2, Epilogue Loop VF:16, Epilogue Loop UF:1
; DEBUG-FORCED: LV: Checking a loop in 'main_vf_vscale_x_2'
; DEBUG-FORCED: LEV: Epilogue vectorization factor is forced.
@@ -204,7 +204,7 @@ exit:
define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
; CHECK-LABEL: @main_vf_vscale_x_2(
; CHECK-NEXT: iter.check:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
@@ -225,25 +225,25 @@ define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
; CHECK-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP7]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 16
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX4]]
-; CHECK-NEXT: store <8 x i64> splat (i64 1), ptr [[TMP9]], align 1
-; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8
+; CHECK-NEXT: store <16 x i64> splat (i64 1), ptr [[TMP9]], align 1
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -343,13 +343,13 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
; CHECK-NEXT: store <vscale x 16 x i8> zeroinitializer, ptr [[TMP6]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 10000, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF7]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF6]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[START]], i64 10000
@@ -357,14 +357,14 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX1]]
-; CHECK-NEXT: store <8 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1
-; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 8
+; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1
+; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 16
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 10000
-; CHECK-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
@@ -466,29 +466,29 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
; CHECK-NEXT: store <vscale x 4 x float> [[TMP13]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF14:![0-9]+]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[N]], 4
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX6]]
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <2 x float>, ptr [[TMP15]], align 4
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP15]], align 4
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX6]]
-; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <2 x float>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = fmul <2 x float> [[WIDE_LOAD7]], [[WIDE_LOAD8]]
-; CHECK-NEXT: store <2 x float> [[TMP17]], ptr [[TMP16]], align 4
-; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 2
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = fmul <4 x float> [[WIDE_LOAD7]], [[WIDE_LOAD8]]
+; CHECK-NEXT: store <4 x float> [[TMP17]], ptr [[TMP16]], align 4
+; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC5]]
-; CHECK-NEXT: br i1 [[TMP18]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP18]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[CMP_N10]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -584,7 +584,7 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
; CHECK-NEXT: store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
index 4cc9244064236..5d9e518fa238d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
@@ -18,16 +18,16 @@ target triple = "aarch64-linux-gnu"
; DEBUG: LV: Checking a loop in 'main_vf_vscale_x_16'
; DEBUG: Create Skeleton for epilogue vectorized loop (first pass)
-; DEBUG: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:8, Epilogue Loop UF:1
+; DEBUG: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:16, Epilogue Loop UF:1
; DEBUG-EPILOG-PREFER-SCALABLE: LV: Checking a loop in 'main_vf_vscale_x_16'
; DEBUG-EPILOG-PREFER-SCALABLE: Create Skeleton for epilogue vectorized loop (first pass)
-; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:vscale x 8, Epilogue Loop UF:1
+; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:vscale x 16, Epilogue Loop UF:1
define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-LABEL: @main_vf_vscale_x_16(
; CHECK-NEXT: iter.check:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N:%.*]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
@@ -53,18 +53,18 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 16
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX1]]
-; CHECK-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP9]], align 1
-; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 8
+; CHECK-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP9]], align 1
+; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 16
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: vec.epilog.middle.block:
@@ -78,10 +78,10 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-LABEL: @main_vf_vscale_x_16(
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: iter.check:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 3
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vector.main.loop.iter.check:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 5
@@ -111,14 +111,14 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.ph:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 3
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP14]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.vector.body:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <vscale x 8 x i8> splat (i8 1), ptr [[TMP15]], align 1
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP15]], align 1
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], [[TMP14]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
index 01494fa91bc66..7106be6ec78dd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -144,15 +144,18 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP12]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP3]], 5
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP11]], 5
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP11]], 3
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
@@ -201,34 +204,38 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP27]], 3
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP33]]
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
; CHECK-NEXT: [[TMP32:%.*]] = sub i64 [[N]], [[N_VEC11]]
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <4 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <vscale x 4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX12]]
; CHECK-NEXT: [[TMP34:%.*]] = load i16, ptr [[SRC]], align 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <4 x i16> poison, i16 [[TMP34]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT14]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP35:%.*]] = sext <4 x i16> [[BROADCAST_SPLAT15]] to <4 x i32>
-; CHECK-NEXT: [[TMP33]] = add <4 x i32> [[VEC_PHI13]], [[TMP35]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP34]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT14]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP40:%.*]] = sext <vscale x 8 x i16> [[BROADCAST_SPLAT15]] to <vscale x 8 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE16]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv8i32(<vscale x 4 x i32> [[VEC_PHI13]], <vscale x 8 x i32> [[TMP40]])
; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i16, ptr [[TMP36]], i64 -3
-; CHECK-NEXT: [[REVERSE17:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLAT15]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: store <4 x i16> [[REVERSE17]], ptr [[TMP38]], align 2
-; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], 4
+; CHECK-NEXT: [[TMP41:%.*]] = sub nuw nsw i64 [[TMP33]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = sub i64 0, [[TMP41]]
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i16, ptr [[TMP36]], i64 [[TMP35]]
+; CHECK-NEXT: [[REVERSE17:%.*]] = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> [[BROADCAST_SPLAT15]])
+; CHECK-NEXT: store <vscale x 8 x i16> [[REVERSE17]], ptr [[TMP42]], align 2
+; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], [[TMP33]]
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT18]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[TMP39]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
+; CHECK-NEXT: [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE16]])
; CHECK-NEXT: [[CMP_N19:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[CMP_N19]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
More information about the llvm-commits
mailing list