[llvm] [LV] Prefer fixed VFs for epilogue vectorization (PR #211814)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 05:57:57 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/211814
>From 83236964b3349504a448b5ae14c87a13e18358e4 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 24 Jul 2026 14:34:49 +0000
Subject: [PATCH 1/2] [LV] Prefer fixed VFs for epilogue vectorization
Unless forced (via `-scalable-vectorization=always`), or if the fixed
vector cost is much higher than scalable vectors (which occurs for some
loops, such as histograms), prefer fixed vectors for the epilogue.
There are currently some hidden costs for scalable epilogues that are
not modeled (such as reduced post-vectorization unrolling).
This restriction likely can be lifted/reworked once we support
predicated vector epilogue.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 15 +++++
.../AArch64/induction-costs-sve.ll | 28 +++++-----
.../AArch64/interleave-with-gaps.ll | 56 ++++++++-----------
.../AArch64/interleaving-load-store.ll | 32 +++++------
.../AArch64/reduction-recurrence-costs-sve.ll | 32 +++++------
.../LoopVectorize/AArch64/store-costs-sve.ll | 22 ++++----
.../AArch64/sve-epilog-vect-vscale-tune.ll | 4 +-
.../AArch64/sve-epilog-vscale-fixed.ll | 19 +++----
.../AArch64/sve2-histcnt-epilogue.ll | 28 +++++-----
.../LoopVectorize/AArch64/sve2-histcnt.ll | 22 ++++----
.../LoopVectorize/AArch64/vector-reverse.ll | 37 +++++-------
11 files changed, 142 insertions(+), 153 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index dbb5ad28fb4ed..f8b2d10346a89 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -706,6 +706,21 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
!B.Width.isScalar())
return true;
+ // Prefer fixed VFs for epilogue loops (unless the fixed cost is much more
+ // expensive -- >= 2x). There are some extra costs for using scalable vectors
+ // in epilogues (e.g., reduced post-vectorization unrolling) that are not
+ // represented in the cost. TODO: Reconsider this restriction for predicated
+ // epilogues (once supported).
+ if (IsEpilogue && A.Width.isScalable() != B.Width.isScalable() &&
+ A.Cost.isValid()) {
+ auto [FixedCost, ScalableCost] = std::make_pair(CostA, CostB);
+ if (B.Width.isFixed())
+ std::swap(FixedCost, ScalableCost);
+
+ if (FixedCost / ScalableCost <= 1)
+ return A.Width.isFixed();
+ }
+
// Improve estimate for the vector width if it is scalable.
unsigned EstimatedWidthA = A.Width.getKnownMinValue();
unsigned EstimatedWidthB = B.Width.getKnownMinValue();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 33c781b99a9bc..c212dba4b3393 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -16,7 +16,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
+; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP7]], 3
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -67,30 +67,30 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP31]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; DEFAULT-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 3
-; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP34]]
+; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP33]], 3
+; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP31]]
; DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP40:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP32:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX9]]
; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 8 x i8>, ptr [[TMP36]], align 1
-; DEFAULT-NEXT: [[TMP32:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
-; DEFAULT-NEXT: [[TMP37:%.*]] = mul <vscale x 8 x i16> [[TMP32]], [[TMP40]]
-; DEFAULT-NEXT: [[TMP38:%.*]] = or <vscale x 8 x i16> [[TMP37]], [[TMP32]]
-; DEFAULT-NEXT: [[TMP35:%.*]] = lshr <vscale x 8 x i16> [[TMP38]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP39:%.*]] = trunc <vscale x 8 x i16> [[TMP35]] to <vscale x 8 x i8>
+; DEFAULT-NEXT: [[TMP34:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP35:%.*]] = mul <vscale x 8 x i16> [[TMP34]], [[TMP32]]
+; DEFAULT-NEXT: [[TMP39:%.*]] = or <vscale x 8 x i16> [[TMP35]], [[TMP34]]
+; DEFAULT-NEXT: [[TMP37:%.*]] = lshr <vscale x 8 x i16> [[TMP39]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i16> [[TMP37]] to <vscale x 8 x i8>
; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX9]]
-; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP39]], ptr [[TMP42]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP34]]
+; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP38]], ptr [[TMP42]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP31]]
; DEFAULT-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC6]]
; DEFAULT-NEXT: br i1 [[TMP43]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -599,11 +599,11 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV]], i32 2
; DEFAULT-NEXT: store i32 0, ptr [[GEP]], align 8
; DEFAULT-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index d64504685fa5d..c47ded7dd10e6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -986,13 +986,10 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br i1 [[CMP]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-NOTF: [[LOOP_PREHEADER]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP47:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP47]]
-; CHECK-NOTF-NEXT: [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = shl nuw i64 [[TMP48]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1078,51 +1075,46 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP51]]
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NOTF-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP50:%.*]] = shl nuw i64 [[TMP49]], 2
-; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], [[TMP50]]
+; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], 4
; CHECK-NOTF-NEXT: [[N_VEC25:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF24]]
; CHECK-NOTF-NEXT: [[TMP46:%.*]] = sub i64 [[N]], [[N_VEC25]]
-; CHECK-NOTF-NEXT: [[TMP52:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NOTF-NEXT: [[TMP47:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
; CHECK-NOTF-NEXT: [[INDEX26:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT35:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <vscale x 4 x i32> [ [[TMP52]], %[[SCALAR_PH]] ], [ [[TMP63:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <4 x i32> [ [[TMP47]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[IV:%.*]] = sub i64 [[N]], [[INDEX26]]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV]]
-; CHECK-NOTF-NEXT: [[TMP55:%.*]] = sub nuw nsw i64 [[TMP50]], 1
-; CHECK-NOTF-NEXT: [[TMP56:%.*]] = mul i64 [[TMP55]], -4
-; CHECK-NOTF-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 [[TMP56]]
-; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <vscale x 16 x i32>, ptr [[TMP57]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC28]])
-; CHECK-NOTF-NEXT: [[TMP58:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 0
-; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP58]])
-; CHECK-NOTF-NEXT: [[TMP59:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 1
-; CHECK-NOTF-NEXT: [[REVERSE31:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP59]])
-; CHECK-NOTF-NEXT: [[TMP60:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 3
-; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP60]])
-; CHECK-NOTF-NEXT: [[TMP61:%.*]] = add <vscale x 4 x i32> [[REVERSE30]], [[VEC_PHI27]]
-; CHECK-NOTF-NEXT: [[TMP62:%.*]] = add <vscale x 4 x i32> [[TMP61]], [[REVERSE31]]
-; CHECK-NOTF-NEXT: [[TMP63]] = add <vscale x 4 x i32> [[TMP62]], [[REVERSE32]]
-; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], [[TMP50]]
+; CHECK-NOTF-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 -12
+; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <16 x i32>, ptr [[TMP50]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC29]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC31]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NOTF-NEXT: [[REVERSE34:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC33]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <4 x i32> [[REVERSE30]], [[VEC_PHI27]]
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <4 x i32> [[TMP51]], [[REVERSE32]]
+; CHECK-NOTF-NEXT: [[TMP53]] = add <4 x i32> [[TMP52]], [[REVERSE34]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], 4
; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT35]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT: [[TMP65:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP63]])
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP53]])
; CHECK-NOTF-NEXT: [[CMP_N36:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N36]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL35:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX36:%.*]] = phi i32 [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL37:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX38:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
; CHECK-NOTF: [[LOOP1]]:
-; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL35]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX36]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL37]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX38]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV1]]
; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[LD1]], [[RDX]]
@@ -1136,7 +1128,7 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[IV1]], 0
; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP1]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: [[RDX_FINAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index 0d195f006cc81..fdba770b168c7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -219,11 +219,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: iter.check:
; INTERLEAVE-2-VLA-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC:%.*]] to i64
; INTERLEAVE-2-VLA-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST:%.*]] to i64
-; INTERLEAVE-2-VLA-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; INTERLEAVE-2-VLA-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; INTERLEAVE-2-VLA: vector.memcheck:
; INTERLEAVE-2-VLA-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
@@ -264,29 +260,27 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.iter.check:
-; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
+; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.ph:
; INTERLEAVE-2-VLA-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; INTERLEAVE-2-VLA-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 2
-; INTERLEAVE-2-VLA-NEXT: [[N_MOD_VF7:%.*]] = urem i64 [[N]], [[TMP17]]
+; INTERLEAVE-2-VLA-NEXT: [[N_MOD_VF7:%.*]] = urem i64 [[N]], 8
; INTERLEAVE-2-VLA-NEXT: [[N_VEC8:%.*]] = sub i64 [[N]], [[N_MOD_VF7]]
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B]], i64 0
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A]], i64 0
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT11]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT9]], <8 x i8> poison, <8 x i32> zeroinitializer
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT11]], <8 x i8> poison, <8 x i32> zeroinitializer
; INTERLEAVE-2-VLA-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.vector.body:
; INTERLEAVE-2-VLA-NEXT: [[INDEX13:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; INTERLEAVE-2-VLA-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX13]]
-; INTERLEAVE-2-VLA-NEXT: [[WIDE_LOAD14:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
-; INTERLEAVE-2-VLA-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i8> [[WIDE_LOAD14]], [[BROADCAST_SPLAT10]]
-; INTERLEAVE-2-VLA-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i8> @llvm.smax.nxv4i8(<vscale x 4 x i8> [[WIDE_LOAD14]], <vscale x 4 x i8> [[BROADCAST_SPLAT12]])
-; INTERLEAVE-2-VLA-NEXT: [[TMP21:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i8> [[BROADCAST_SPLAT10]], <vscale x 4 x i8> [[TMP20]]
+; INTERLEAVE-2-VLA-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i8>, ptr [[TMP18]], align 1
+; INTERLEAVE-2-VLA-NEXT: [[TMP17:%.*]] = icmp sgt <8 x i8> [[WIDE_LOAD14]], [[BROADCAST_SPLAT10]]
+; INTERLEAVE-2-VLA-NEXT: [[TMP19:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[WIDE_LOAD14]], <8 x i8> [[BROADCAST_SPLAT12]])
+; INTERLEAVE-2-VLA-NEXT: [[TMP16:%.*]] = select <8 x i1> [[TMP17]], <8 x i8> [[BROADCAST_SPLAT10]], <8 x i8> [[TMP19]]
; INTERLEAVE-2-VLA-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX13]]
-; INTERLEAVE-2-VLA-NEXT: store <vscale x 4 x i8> [[TMP21]], ptr [[TMP22]], align 1
-; INTERLEAVE-2-VLA-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX13]], [[TMP17]]
+; INTERLEAVE-2-VLA-NEXT: store <8 x i8> [[TMP16]], ptr [[TMP22]], align 1
+; INTERLEAVE-2-VLA-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX13]], 8
; INTERLEAVE-2-VLA-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC8]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[TMP23]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 33f6ce0766e96..15f1103dbca33 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -607,8 +607,8 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
-; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
+; VSCALEFORTUNING2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -657,36 +657,36 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
+; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
-; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP20]]
+; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[TMP32:%.*]] = shl nuw i64 [[TMP24]], 2
+; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP32]]
; VSCALEFORTUNING2-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
; VSCALEFORTUNING2-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_VECTOR_BODY]]:
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; VSCALEFORTUNING2-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
-; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD10]], [[BROADCAST_SPLAT7]]
-; VSCALEFORTUNING2-NEXT: [[TMP21]] = or <vscale x 4 x i16> [[TMP24]], [[VEC_PHI9]]
-; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP20]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
+; VSCALEFORTUNING2-NEXT: [[TMP34:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
+; VSCALEFORTUNING2-NEXT: [[TMP33]] = or <vscale x 4 x i16> [[TMP34]], [[VEC_PHI15]]
+; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP32]]
; VSCALEFORTUNING2-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP21]])
+; VSCALEFORTUNING2-NEXT: [[TMP35:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP33]])
; VSCALEFORTUNING2-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; VSCALEFORTUNING2: [[VEC_EPILOG_SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP35]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
; VSCALEFORTUNING2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -699,7 +699,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP35]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; PRED-LABEL: define i16 @reduce_udiv(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 1b46ee158117c..a54ae51a5a56c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -10,10 +10,10 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: iter.check:
; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP10]], 2
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP13]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP6]], 2
+; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP10]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP6]]
+; DEFAULT-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP2]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; DEFAULT: vector.main.loop.iter.check:
; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 5
@@ -38,22 +38,22 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; DEFAULT: vec.epilog.iter.check:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP15]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: vec.epilog.ph:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 2
-; DEFAULT-NEXT: [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP12]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP14]], 2
+; DEFAULT-NEXT: [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP11]]
; DEFAULT-NEXT: [[N_VEC3:%.*]] = sub i64 101, [[N_MOD_VF2]]
; DEFAULT-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; DEFAULT: vec.epilog.vector.body:
; DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX1]]
; DEFAULT-NEXT: store <vscale x 4 x i8> zeroinitializer, ptr [[TMP9]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP12]]
-; DEFAULT-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
-; DEFAULT-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP11]]
+; DEFAULT-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
+; DEFAULT-NEXT: br i1 [[TMP13]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: vec.epilog.middle.block:
; DEFAULT-NEXT: [[CMP_N6:%.*]] = icmp eq i64 101, [[N_VEC3]]
; DEFAULT-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
index 5196b96efb9a2..4d00ce7643fbd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
@@ -12,10 +12,10 @@ define void @foo(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i6
; CHECK-EPILOG: vec.epilog.vector.body:
; CHECK-EPILOG: load <8 x i16>
-; The epilogue loop gets vectorised vscale x 2 x i16 wide.
+; The epilogue loop gets vectorised 4 x i16 wide.
; CHECK-EPILOG-V2: vec.epilog.ph:
; CHECK-EPILOG-V2: vec.epilog.vector.body:
-; CHECK-EPILOG-V2: load <vscale x 2 x i16>
+; CHECK-EPILOG-V2: load <4 x i16>
; CHECK-NO-EPILOG-NOT: vec.epilog.vector.ph:
; CHECK-NO-EPILOG-NOT: vec.epilog.vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
index 5d9e518fa238d..d5870a2978b00 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
@@ -22,7 +22,7 @@ target triple = "aarch64-linux-gnu"
; DEBUG-EPILOG-PREFER-SCALABLE: LV: Checking a loop in 'main_vf_vscale_x_16'
; DEBUG-EPILOG-PREFER-SCALABLE: Create Skeleton for epilogue vectorized loop (first pass)
-; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:vscale x 16, Epilogue Loop UF:1
+; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:16, Epilogue Loop UF:1
define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-LABEL: @main_vf_vscale_x_16(
@@ -77,13 +77,10 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
;
; CHECK-EPILOG-PREFER-SCALABLE-LABEL: @main_vf_vscale_x_16(
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vector.main.loop.iter.check:
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 5
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -106,20 +103,18 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP11]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.ph:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 4
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP14]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.vector.body:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP15]], align 1
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], [[TMP14]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP15]], align 1
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 50942fc5adafb..a3ee22856af4e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -8,13 +8,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: iter.check:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
@@ -37,23 +34,26 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP25]]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i32>, ptr [[TMP18]], align 4
-; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD5]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 2 x i64> [[TMP19]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> [[TMP20]], i32 1, <vscale x 2 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP25]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x i32>, ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = zext <2 x i32> [[WIDE_LOAD5]] to <2 x i64>
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i64> [[TMP16]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP16]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x ptr> [[TMP12]], ptr [[TMP19]], i32 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.v2p0.i32(<2 x ptr> [[TMP13]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 2ec48b25af5b5..70240d7b20b96 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -239,10 +239,10 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: iter.check:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP5]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP2]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 4
@@ -267,23 +267,23 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]]
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP7]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP12]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP9]], 3
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP11]]
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP14]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[TMP15]], i8 1, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP12]]
+; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
+; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP12]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[WIDE_GEP6]], i8 1, <vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP11]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
index 7106be6ec78dd..770ce345af7bb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -144,13 +144,10 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP12]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP11]], 5
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -204,38 +201,34 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP14]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP27]], 3
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP33]]
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], 8
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
; CHECK-NEXT: [[TMP32:%.*]] = sub i64 [[N]], [[N_VEC11]]
-; CHECK-NEXT: [[TMP37:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <vscale x 4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <8 x i32> [ [[TMP24]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX12]]
; CHECK-NEXT: [[TMP34:%.*]] = load i16, ptr [[SRC]], align 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP34]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT14]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP40:%.*]] = sext <vscale x 8 x i16> [[BROADCAST_SPLAT15]] to <vscale x 8 x i32>
-; CHECK-NEXT: [[PARTIAL_REDUCE16]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv8i32(<vscale x 4 x i32> [[VEC_PHI13]], <vscale x 8 x i32> [[TMP40]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <8 x i16> poison, i16 [[TMP34]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT14]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = sext <8 x i16> [[BROADCAST_SPLAT15]] to <8 x i32>
+; CHECK-NEXT: [[TMP33]] = add <8 x i32> [[VEC_PHI13]], [[TMP27]]
; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP41:%.*]] = sub nuw nsw i64 [[TMP33]], 1
-; CHECK-NEXT: [[TMP35:%.*]] = sub i64 0, [[TMP41]]
-; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i16, ptr [[TMP36]], i64 [[TMP35]]
-; CHECK-NEXT: [[REVERSE17:%.*]] = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> [[BROADCAST_SPLAT15]])
-; CHECK-NEXT: store <vscale x 8 x i16> [[REVERSE17]], ptr [[TMP42]], align 2
-; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], [[TMP33]]
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i16, ptr [[TMP36]], i64 -7
+; CHECK-NEXT: [[REVERSE16:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLAT15]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: store <8 x i16> [[REVERSE16]], ptr [[TMP35]], align 2
+; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], 8
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT18]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[TMP39]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-NEXT: [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP33]])
; CHECK-NEXT: [[CMP_N19:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[CMP_N19]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
>From ed3cc03f37892ffdc4e38892e78513f05d360f19 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 31 Jul 2026 12:55:09 +0000
Subject: [PATCH 2/2] Fixups
---
.../AArch64/induction-costs-sve.ll | 28 ++++++++--------
.../AArch64/reduction-recurrence-costs-sve.ll | 32 +++++++++----------
.../LoopVectorize/AArch64/store-costs-sve.ll | 22 ++++++-------
.../LoopVectorize/AArch64/sve2-histcnt.ll | 22 ++++++-------
4 files changed, 52 insertions(+), 52 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index c212dba4b3393..33c781b99a9bc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -16,7 +16,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP7]], 3
+; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -67,30 +67,30 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP31]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; DEFAULT-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP33]], 3
-; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP31]]
+; DEFAULT-NEXT: [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 3
+; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP34]]
; DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP32:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP40:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX9]]
; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 8 x i8>, ptr [[TMP36]], align 1
-; DEFAULT-NEXT: [[TMP34:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
-; DEFAULT-NEXT: [[TMP35:%.*]] = mul <vscale x 8 x i16> [[TMP34]], [[TMP32]]
-; DEFAULT-NEXT: [[TMP39:%.*]] = or <vscale x 8 x i16> [[TMP35]], [[TMP34]]
-; DEFAULT-NEXT: [[TMP37:%.*]] = lshr <vscale x 8 x i16> [[TMP39]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i16> [[TMP37]] to <vscale x 8 x i8>
+; DEFAULT-NEXT: [[TMP32:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP37:%.*]] = mul <vscale x 8 x i16> [[TMP32]], [[TMP40]]
+; DEFAULT-NEXT: [[TMP38:%.*]] = or <vscale x 8 x i16> [[TMP37]], [[TMP32]]
+; DEFAULT-NEXT: [[TMP35:%.*]] = lshr <vscale x 8 x i16> [[TMP38]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP39:%.*]] = trunc <vscale x 8 x i16> [[TMP35]] to <vscale x 8 x i8>
; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX9]]
-; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP38]], ptr [[TMP42]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP31]]
+; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP39]], ptr [[TMP42]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP34]]
; DEFAULT-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC6]]
; DEFAULT-NEXT: br i1 [[TMP43]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -599,11 +599,11 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[IV_CONV]], i32 2
; DEFAULT-NEXT: store i32 0, ptr [[GEP]], align 8
; DEFAULT-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 15f1103dbca33..33f6ce0766e96 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -607,8 +607,8 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
-; VSCALEFORTUNING2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
+; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -657,36 +657,36 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
+; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP32:%.*]] = shl nuw i64 [[TMP24]], 2
-; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP32]]
+; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
+; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP20]]
; VSCALEFORTUNING2-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
; VSCALEFORTUNING2-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_VECTOR_BODY]]:
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; VSCALEFORTUNING2-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
-; VSCALEFORTUNING2-NEXT: [[TMP34:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
-; VSCALEFORTUNING2-NEXT: [[TMP33]] = or <vscale x 4 x i16> [[TMP34]], [[VEC_PHI15]]
-; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP32]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
+; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD10]], [[BROADCAST_SPLAT7]]
+; VSCALEFORTUNING2-NEXT: [[TMP21]] = or <vscale x 4 x i16> [[TMP24]], [[VEC_PHI9]]
+; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP20]]
; VSCALEFORTUNING2-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; VSCALEFORTUNING2-NEXT: [[TMP35:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP33]])
+; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP21]])
; VSCALEFORTUNING2-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; VSCALEFORTUNING2: [[VEC_EPILOG_SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP35]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
; VSCALEFORTUNING2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -699,7 +699,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP35]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; PRED-LABEL: define i16 @reduce_udiv(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index a54ae51a5a56c..1b46ee158117c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -10,10 +10,10 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: iter.check:
; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP10]], 2
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP6]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP2]], 2
+; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP10]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP13]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP6]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; DEFAULT: vector.main.loop.iter.check:
; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 5
@@ -38,22 +38,22 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; DEFAULT: vec.epilog.iter.check:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP12]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP15]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: vec.epilog.ph:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP14]], 2
-; DEFAULT-NEXT: [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP11]]
+; DEFAULT-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 2
+; DEFAULT-NEXT: [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP12]]
; DEFAULT-NEXT: [[N_VEC3:%.*]] = sub i64 101, [[N_MOD_VF2]]
; DEFAULT-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; DEFAULT: vec.epilog.vector.body:
; DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX1]]
; DEFAULT-NEXT: store <vscale x 4 x i8> zeroinitializer, ptr [[TMP9]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP11]]
-; DEFAULT-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
-; DEFAULT-NEXT: br i1 [[TMP13]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP12]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
+; DEFAULT-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: vec.epilog.middle.block:
; DEFAULT-NEXT: [[CMP_N6:%.*]] = icmp eq i64 101, [[N_VEC3]]
; DEFAULT-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 70240d7b20b96..2ec48b25af5b5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -239,10 +239,10 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: iter.check:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP5]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 3
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP2]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 4
@@ -267,23 +267,23 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]]
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP3]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP7]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP9]], 3
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP11]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP12]]
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP12]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[WIDE_GEP6]], i8 1, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP11]]
+; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 8 x i32> [[WIDE_LOAD5]] to <vscale x 8 x i64>
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 8 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv8p0.i8(<vscale x 8 x ptr> [[TMP15]], i8 1, <vscale x 8 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP12]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: vec.epilog.middle.block:
More information about the llvm-commits
mailing list