[llvm] 11eba5c - [LV] Prefer fixed VFs for epilogue vectorization (#211814)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 02:22:47 PDT 2026
Author: Benjamin Maxwell
Date: 2026-08-12T10:22:42+01:00
New Revision: 11eba5ca3511b1039d5e672b3d886bd5b9fc5807
URL: https://github.com/llvm/llvm-project/commit/11eba5ca3511b1039d5e672b3d886bd5b9fc5807
DIFF: https://github.com/llvm/llvm-project/commit/11eba5ca3511b1039d5e672b3d886bd5b9fc5807.diff
LOG: [LV] Prefer fixed VFs for epilogue vectorization (#211814)
Unless forced (via `-scalable-vectorization=always`), or if the fixed
vector cost is much higher than scalable vectors (which occurs for some
loops, such as histograms), prefer fixed vectors for the epilogue.
There are currently some hidden costs for scalable epilogues that are
not modeled (such as reduced post-vectorization unrolling).
This restriction likely can be lifted/reworked once we support
predicated vector epilogue.
Added:
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..973ed8820691d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1944,8 +1944,7 @@ class TargetTransformInfo {
/// \returns True if the target prefers fixed width vectorization if the
/// loop vectorizer's cost-model assigns an equal cost to the fixed and
/// scalable version of the vectorized loop.
- /// \p IsEpilogue is true if the decision is for the epilogue loop.
- LLVM_ABI bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const;
+ LLVM_ABI bool preferFixedOverScalableIfEqualCost() const;
/// \returns True if target prefers SLP vectorizer with altermate opcode
/// vectorization, false - otherwise.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 25574a2d2c0e5..1cda5ff84d29f 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1159,9 +1159,7 @@ class LLVM_ABI TargetTransformInfoImplBase {
return VF;
}
- virtual bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const {
- return false;
- }
+ virtual bool preferFixedOverScalableIfEqualCost() const { return false; }
virtual bool preferInLoopReduction(RecurKind Kind, Type *Ty) const {
return false;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 595c2d10dd118..42afdf67a5f93 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1468,9 +1468,8 @@ unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
}
-bool TargetTransformInfo::preferFixedOverScalableIfEqualCost(
- bool IsEpilogue) const {
- return TTIImpl->preferFixedOverScalableIfEqualCost(IsEpilogue);
+bool TargetTransformInfo::preferFixedOverScalableIfEqualCost() const {
+ return TTIImpl->preferFixedOverScalableIfEqualCost();
}
bool TargetTransformInfo::preferInLoopReduction(RecurKind Kind,
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 21a9e752cad24..c21fd5fcdf6c0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -7227,15 +7227,9 @@ static bool containsDecreasingPointers(Loop *TheLoop,
return false;
}
-bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost(bool IsEpilogue) const {
+bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost() const {
if (SVEPreferFixedOverScalableIfEqualCost.getNumOccurrences())
return SVEPreferFixedOverScalableIfEqualCost;
- // For cases like post-LTO vectorization, when we eventually know the trip
- // count, epilogue with fixed-width vectorization can be deleted if the trip
- // count is less than the epilogue iterations. That's why we prefer
- // fixed-width vectorization in epilogue in case of equal costs.
- if (IsEpilogue)
- return true;
return ST->useFixedOverScalableIfEqualCost();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ddca341bad05b..cd71b1179bd45 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -465,7 +465,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
: TailFoldingStyle::DataWithoutLaneMask;
}
- bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const override;
+ bool preferFixedOverScalableIfEqualCost() const override;
unsigned getEpilogueVectorizationMinVF() const override;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index fd7fd2e011a83..ff9b9171d8c8c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -67,6 +67,10 @@ static cl::opt<bool> ForceTargetSupportsGatherScatterOps(
cl::desc("Assume the target supports gather/scatter operations (used for "
"testing)."));
+static cl::opt<float> ScalableEpilogueVFCostScaleFactor(
+ "scalable-epilogue-vf-cost-scale-factor", cl::init(2.0), cl::Hidden,
+ cl::desc("Scale the cost of scalable epilogue VFs by this factor."));
+
/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
/// is passed, the message relates to that particular instruction.
#ifndef NDEBUG
@@ -702,6 +706,23 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
!B.Width.isScalar())
return true;
+ // Favor fixed VFs for epilogue loops by scaling the costs of scalable VFs
+ // 'ScalableEpilogueVFCostScaleFactor' (default 2.0). This is intended to
+ // model that fixed VFs are more likely to be fully unrolled (or optimized
+ // out) post vectorization. TODO: Reconsider this restriction for predicated
+ // epilogues (once supported).
+ if (IsEpilogue && A.Width.isScalable() != B.Width.isScalable() &&
+ A.Cost.isValid() && B.Cost.isValid()) {
+ auto [FixedCost, ScalableCost] = std::make_pair(CostA, CostB);
+ if (B.Width.isFixed())
+ std::swap(FixedCost, ScalableCost);
+
+ ScalableCost *= ScalableEpilogueVFCostScaleFactor;
+
+ if (FixedCost <= ScalableCost)
+ return A.Width.isFixed();
+ }
+
// Improve estimate for the vector width if it is scalable.
unsigned EstimatedWidthA = A.Width.getKnownMinValue();
unsigned EstimatedWidthB = B.Width.getKnownMinValue();
@@ -722,7 +743,7 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
// Assume vscale may be larger than 1 (or the value being tuned for),
// so that scalable vectorization is slightly favorable over fixed-width
// vectorization.
- bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost(IsEpilogue) &&
+ bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost() &&
A.Width.isScalable() && !B.Width.isScalable();
auto CmpFn = [PreferScalable](const InstructionCost &LHS,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index da4d004a45637..898a5d9e8a461 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -12,11 +12,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
; DEFAULT-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -27,6 +23,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP27]]
; DEFAULT-NEXT: br i1 [[DIFF_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP1]], 4
; DEFAULT-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], [[TMP8]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK3]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -66,30 +63,28 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP31]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 3
-; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[TMP0]], [[TMP34]]
+; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = and i64 [[TMP0]], 7
; DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP40:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT8]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT7]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP28:%.*]] = trunc <8 x i32> [[BROADCAST_SPLAT8]] to <8 x i16>
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX9]]
-; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 8 x i8>, ptr [[TMP36]], align 1
-; DEFAULT-NEXT: [[TMP32:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD10]] to <vscale x 8 x i16>
-; DEFAULT-NEXT: [[TMP37:%.*]] = mul <vscale x 8 x i16> [[TMP32]], [[TMP40]]
-; DEFAULT-NEXT: [[TMP38:%.*]] = or <vscale x 8 x i16> [[TMP37]], [[TMP32]]
-; DEFAULT-NEXT: [[TMP35:%.*]] = lshr <vscale x 8 x i16> [[TMP38]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP39:%.*]] = trunc <vscale x 8 x i16> [[TMP35]] to <vscale x 8 x i8>
+; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP36]], align 1
+; DEFAULT-NEXT: [[TMP34:%.*]] = zext <8 x i8> [[WIDE_LOAD10]] to <8 x i16>
+; DEFAULT-NEXT: [[TMP35:%.*]] = mul <8 x i16> [[TMP34]], [[TMP28]]
+; DEFAULT-NEXT: [[TMP31:%.*]] = or <8 x i16> [[TMP35]], [[TMP34]]
+; DEFAULT-NEXT: [[TMP32:%.*]] = lshr <8 x i16> [[TMP31]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP33:%.*]] = trunc <8 x i16> [[TMP32]] to <8 x i8>
; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX9]]
-; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP39]], ptr [[TMP42]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], [[TMP34]]
+; DEFAULT-NEXT: store <8 x i8> [[TMP33]], ptr [[TMP42]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], 8
; DEFAULT-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC6]]
; DEFAULT-NEXT: br i1 [[TMP43]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 2733930833e7e..9b37d5280e262 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -986,13 +986,10 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br i1 [[CMP]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK-NOTF: [[LOOP_PREHEADER]]:
; CHECK-NOTF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP47:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP47]]
-; CHECK-NOTF-NEXT: [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = shl nuw i64 [[TMP48]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1078,51 +1075,46 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP51]]
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NOTF-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP50:%.*]] = shl nuw i64 [[TMP49]], 2
-; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = urem i64 [[TMP0]], [[TMP50]]
+; CHECK-NOTF-NEXT: [[N_MOD_VF24:%.*]] = and i64 [[TMP0]], 3
; CHECK-NOTF-NEXT: [[N_VEC25:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF24]]
; CHECK-NOTF-NEXT: [[TMP46:%.*]] = sub i64 [[N]], [[N_VEC25]]
-; CHECK-NOTF-NEXT: [[TMP52:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NOTF-NEXT: br label %[[LOOP:.*]]
; CHECK-NOTF: [[LOOP]]:
; CHECK-NOTF-NEXT: [[INDEX26:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT35:%.*]], %[[LOOP]] ]
-; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <vscale x 4 x i32> [ [[TMP52]], %[[SCALAR_PH]] ], [ [[TMP63:%.*]], %[[LOOP]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI27:%.*]] = phi <4 x i32> [ [[TMP48]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[IV:%.*]] = sub i64 [[N]], [[INDEX26]]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV]]
-; CHECK-NOTF-NEXT: [[TMP55:%.*]] = sub nuw nsw i64 [[TMP50]], 1
-; CHECK-NOTF-NEXT: [[TMP56:%.*]] = mul i64 [[TMP55]], -4
-; CHECK-NOTF-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 [[TMP56]]
-; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <vscale x 16 x i32>, ptr [[TMP57]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC28]])
-; CHECK-NOTF-NEXT: [[TMP58:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 0
-; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP58]])
-; CHECK-NOTF-NEXT: [[TMP59:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 1
-; CHECK-NOTF-NEXT: [[REVERSE31:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP59]])
-; CHECK-NOTF-NEXT: [[TMP60:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC29]], 3
-; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP60]])
-; CHECK-NOTF-NEXT: [[TMP61:%.*]] = add <vscale x 4 x i32> [[REVERSE30]], [[VEC_PHI27]]
-; CHECK-NOTF-NEXT: [[TMP62:%.*]] = add <vscale x 4 x i32> [[TMP61]], [[REVERSE31]]
-; CHECK-NOTF-NEXT: [[TMP63]] = add <vscale x 4 x i32> [[TMP62]], [[REVERSE32]]
-; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], [[TMP50]]
+; CHECK-NOTF-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i64 -12
+; CHECK-NOTF-NEXT: [[WIDE_VEC28:%.*]] = load <16 x i32>, ptr [[TMP50]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NOTF-NEXT: [[REVERSE30:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC29]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NOTF-NEXT: [[REVERSE32:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC31]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <16 x i32> [[WIDE_VEC28]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NOTF-NEXT: [[REVERSE34:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC33]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <4 x i32> [[REVERSE30]], [[VEC_PHI27]]
+; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <4 x i32> [[TMP51]], [[REVERSE32]]
+; CHECK-NOTF-NEXT: [[TMP53]] = add <4 x i32> [[TMP52]], [[REVERSE34]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT35]] = add nuw i64 [[INDEX26]], 4
; CHECK-NOTF-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT35]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[TMP54]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK-NOTF: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT: [[TMP65:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP63]])
+; CHECK-NOTF-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP53]])
; CHECK-NOTF-NEXT: [[CMP_N36:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC25]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N36]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NOTF-NEXT: [[BC_RESUME_VAL35:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX36:%.*]] = phi i32 [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL36:%.*]] = phi i64 [ [[TMP46]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[LOOP_PREHEADER]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX37:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP45]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[LOOP_PREHEADER]] ]
; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
; CHECK-NOTF: [[LOOP1]]:
-; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL35]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX36]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_RESUME_VAL36]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[RDX_NEXT:%.*]], %[[LOOP1]] ], [ [[BC_MERGE_RDX37]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [16 x i8], ptr [[A]], i64 [[IV1]]
; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[LD1]], [[RDX]]
@@ -1136,7 +1128,7 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[IV1]], 0
; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[EXIT_LOOPEXIT]], label %[[LOOP1]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK-NOTF: [[EXIT_LOOPEXIT]]:
-; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP65]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP1]] ], [ [[TMP45]], %[[MIDDLE_BLOCK]] ], [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NOTF-NEXT: br label %[[EXIT]]
; CHECK-NOTF: [[EXIT]]:
; CHECK-NOTF-NEXT: [[RDX_FINAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_NEXT_LCSSA]], %[[EXIT_LOOPEXIT]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index 05347656768b8..5c321b729ac7c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -219,11 +219,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: iter.check:
; INTERLEAVE-2-VLA-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC:%.*]] to i64
; INTERLEAVE-2-VLA-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST:%.*]] to i64
-; INTERLEAVE-2-VLA-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; INTERLEAVE-2-VLA-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; INTERLEAVE-2-VLA: vector.memcheck:
; INTERLEAVE-2-VLA-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
@@ -264,29 +260,27 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.iter.check:
-; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
+; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.ph:
; INTERLEAVE-2-VLA-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; INTERLEAVE-2-VLA-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 2
-; INTERLEAVE-2-VLA-NEXT: [[N_MOD_VF7:%.*]] = urem i64 [[N]], [[TMP17]]
+; INTERLEAVE-2-VLA-NEXT: [[N_MOD_VF7:%.*]] = and i64 [[N]], 7
; INTERLEAVE-2-VLA-NEXT: [[N_VEC8:%.*]] = sub i64 [[N]], [[N_MOD_VF7]]
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B]], i64 0
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A]], i64 0
-; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT11]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <8 x i8> poison, i8 [[B]], i64 0
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT9]], <8 x i8> poison, <8 x i32> zeroinitializer
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i8> poison, i8 [[A]], i64 0
+; INTERLEAVE-2-VLA-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT11]], <8 x i8> poison, <8 x i32> zeroinitializer
; INTERLEAVE-2-VLA-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.vector.body:
; INTERLEAVE-2-VLA-NEXT: [[INDEX13:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; INTERLEAVE-2-VLA-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX13]]
-; INTERLEAVE-2-VLA-NEXT: [[WIDE_LOAD14:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
-; INTERLEAVE-2-VLA-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i8> [[WIDE_LOAD14]], [[BROADCAST_SPLAT10]]
-; INTERLEAVE-2-VLA-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i8> @llvm.smax.nxv4i8(<vscale x 4 x i8> [[WIDE_LOAD14]], <vscale x 4 x i8> [[BROADCAST_SPLAT12]])
-; INTERLEAVE-2-VLA-NEXT: [[TMP21:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i8> [[BROADCAST_SPLAT10]], <vscale x 4 x i8> [[TMP20]]
+; INTERLEAVE-2-VLA-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i8>, ptr [[TMP18]], align 1
+; INTERLEAVE-2-VLA-NEXT: [[TMP17:%.*]] = icmp sgt <8 x i8> [[WIDE_LOAD14]], [[BROADCAST_SPLAT10]]
+; INTERLEAVE-2-VLA-NEXT: [[TMP19:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[WIDE_LOAD14]], <8 x i8> [[BROADCAST_SPLAT12]])
+; INTERLEAVE-2-VLA-NEXT: [[TMP16:%.*]] = select <8 x i1> [[TMP17]], <8 x i8> [[BROADCAST_SPLAT10]], <8 x i8> [[TMP19]]
; INTERLEAVE-2-VLA-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX13]]
-; INTERLEAVE-2-VLA-NEXT: store <vscale x 4 x i8> [[TMP21]], ptr [[TMP22]], align 1
-; INTERLEAVE-2-VLA-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX13]], [[TMP17]]
+; INTERLEAVE-2-VLA-NEXT: store <8 x i8> [[TMP16]], ptr [[TMP22]], align 1
+; INTERLEAVE-2-VLA-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX13]], 8
; INTERLEAVE-2-VLA-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC8]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[TMP23]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index a77306354358c..132d47a12a7c8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -502,13 +502,10 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-SAME: ptr [[SRC:%.*]], i16 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 4
; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP7]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
@@ -555,40 +552,38 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; DEFAULT-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; DEFAULT-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 2
-; DEFAULT-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP25]]
+; DEFAULT-NEXT: [[N_MOD_VF10:%.*]] = and i64 [[TMP0]], 3
; DEFAULT-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
-; DEFAULT-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT12]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP25:%.*]] = insertelement <4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i16> poison, i16 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT11]], <4 x i16> poison, <4 x i32> zeroinitializer
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT: [[VEC_PHI15:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP29:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[VEC_PHI15:%.*]] = phi <4 x i16> [ [[TMP25]], %[[VEC_EPILOG_PH]] ], [ [[TMP24:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
-; DEFAULT-NEXT: [[TMP28:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
-; DEFAULT-NEXT: [[TMP29]] = or <vscale x 4 x i16> [[TMP28]], [[VEC_PHI15]]
-; DEFAULT-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[IV]], [[TMP25]]
+; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <4 x i16>, ptr [[GEP]], align 2
+; DEFAULT-NEXT: [[TMP28:%.*]] = udiv <4 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
+; DEFAULT-NEXT: [[TMP24]] = or <4 x i16> [[TMP28]], [[VEC_PHI15]]
+; DEFAULT-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[IV]], 4
; DEFAULT-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC11]]
; DEFAULT-NEXT: br i1 [[TMP30]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[TMP31:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP29]])
+; DEFAULT-NEXT: [[TMP26:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP24]])
; DEFAULT-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; DEFAULT-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; DEFAULT-NEXT: [[BC_MERGE_RDX18:%.*]] = phi i16 [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP33]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; DEFAULT-NEXT: br label %[[LOOP1:.*]]
; DEFAULT: [[LOOP1]]:
; DEFAULT-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; DEFAULT-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX19]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP1]] ]
+; DEFAULT-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX18]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP1]] ]
; DEFAULT-NEXT: [[GEP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV1]]
; DEFAULT-NEXT: [[L:%.*]] = load i16, ptr [[GEP1]], align 2
; DEFAULT-NEXT: [[DIV:%.*]] = udiv i16 [[L]], [[X]]
@@ -597,20 +592,17 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; DEFAULT: [[EXIT]]:
-; DEFAULT-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP1]] ], [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; DEFAULT-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP1]] ], [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; DEFAULT-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; VSCALEFORTUNING2-LABEL: define i16 @reduce_udiv(
; VSCALEFORTUNING2-SAME: ptr [[SRC:%.*]], i16 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; VSCALEFORTUNING2-NEXT: [[ITER_CHECK:.*]]:
; VSCALEFORTUNING2-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
-; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
-; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
+; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 8
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -657,40 +649,38 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
+; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
-; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], [[TMP20]]
+; VSCALEFORTUNING2-NEXT: [[N_MOD_VF4:%.*]] = and i64 [[TMP0]], 7
; VSCALEFORTUNING2-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
-; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = insertelement <vscale x 4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i64 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
-; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT: [[TMP32:%.*]] = insertelement <8 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i64 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i16> poison, i16 [[X]], i64 0
+; VSCALEFORTUNING2-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT11]], <8 x i16> poison, <8 x i32> zeroinitializer
; VSCALEFORTUNING2-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_VECTOR_BODY]]:
; VSCALEFORTUNING2-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; VSCALEFORTUNING2-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 4 x i16> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; VSCALEFORTUNING2-NEXT: [[VEC_PHI15:%.*]] = phi <8 x i16> [ [[TMP32]], %[[VEC_EPILOG_PH]] ], [ [[TMP24:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; VSCALEFORTUNING2-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i16>, ptr [[GEP]], align 2
-; VSCALEFORTUNING2-NEXT: [[TMP24:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD10]], [[BROADCAST_SPLAT7]]
-; VSCALEFORTUNING2-NEXT: [[TMP21]] = or <vscale x 4 x i16> [[TMP24]], [[VEC_PHI9]]
-; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], [[TMP20]]
+; VSCALEFORTUNING2-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[GEP]], align 2
+; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = udiv <8 x i16> [[WIDE_LOAD16]], [[BROADCAST_SPLAT13]]
+; VSCALEFORTUNING2-NEXT: [[TMP24]] = or <8 x i16> [[TMP23]], [[VEC_PHI15]]
+; VSCALEFORTUNING2-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[IV]], 8
; VSCALEFORTUNING2-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[TMP25]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; VSCALEFORTUNING2-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP21]])
+; VSCALEFORTUNING2-NEXT: [[TMP26:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP24]])
; VSCALEFORTUNING2-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; VSCALEFORTUNING2: [[VEC_EPILOG_SCALAR_PH]]:
; VSCALEFORTUNING2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i16 [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; VSCALEFORTUNING2-NEXT: [[BC_MERGE_RDX18:%.*]] = phi i16 [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; VSCALEFORTUNING2-NEXT: br label %[[LOOP:.*]]
; VSCALEFORTUNING2: [[LOOP]]:
; VSCALEFORTUNING2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; VSCALEFORTUNING2-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX19]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; VSCALEFORTUNING2-NEXT: [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX18]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
; VSCALEFORTUNING2-NEXT: [[GEP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV1]]
; VSCALEFORTUNING2-NEXT: [[L:%.*]] = load i16, ptr [[GEP1]], align 2
; VSCALEFORTUNING2-NEXT: [[DIV:%.*]] = udiv i16 [[L]], [[X]]
@@ -699,7 +689,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
; VSCALEFORTUNING2-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; VSCALEFORTUNING2: [[EXIT]]:
-; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VSCALEFORTUNING2-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i16 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP31]], %[[MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; VSCALEFORTUNING2-NEXT: ret i16 [[RED_NEXT_LCSSA]]
;
; PRED-LABEL: define i16 @reduce_udiv(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 1b46ee158117c..c57a56b09d3c4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -9,13 +9,9 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-LABEL: define void @cost_store_i8(
; DEFAULT-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: iter.check:
-; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP10]], 2
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP13]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP6]], 2
-; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; DEFAULT-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; DEFAULT: vector.main.loop.iter.check:
+; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 5
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP1]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -38,27 +34,22 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; DEFAULT: vec.epilog.iter.check:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP15]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: vec.epilog.ph:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 2
-; DEFAULT-NEXT: [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP12]]
-; DEFAULT-NEXT: [[N_VEC3:%.*]] = sub i64 101, [[N_MOD_VF2]]
; DEFAULT-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; DEFAULT: vec.epilog.vector.body:
; DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX1]]
-; DEFAULT-NEXT: store <vscale x 4 x i8> zeroinitializer, ptr [[TMP9]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP12]]
-; DEFAULT-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC3]]
-; DEFAULT-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT: store <4 x i8> zeroinitializer, ptr [[TMP9]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; DEFAULT-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
+; DEFAULT-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: vec.epilog.middle.block:
-; DEFAULT-NEXT: [[CMP_N6:%.*]] = icmp eq i64 101, [[N_VEC3]]
-; DEFAULT-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; DEFAULT-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; DEFAULT: vec.epilog.scalar.ph:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
; DEFAULT-NEXT: br label [[LOOP:%.*]]
; DEFAULT: loop:
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
index 5196b96efb9a2..32895d02f091d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
@@ -4,6 +4,8 @@
; RUN: -mcpu=neoverse-v2 < %s | FileCheck %s --check-prefix=CHECK-EPILOG-V2
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 \
; RUN: -mcpu=cortex-x2 < %s | FileCheck %s --check-prefix=CHECK-NO-EPILOG
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -scalable-epilogue-vf-cost-scale-factor=0.9 \
+; RUN: -mcpu=neoverse-v2 < %s | FileCheck %s --check-prefix=CHECK-EPILOG-V2-PREF-SCALBLE
target triple = "aarch64-unknown-linux-gnu"
@@ -12,10 +14,15 @@ define void @foo(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i6
; CHECK-EPILOG: vec.epilog.vector.body:
; CHECK-EPILOG: load <8 x i16>
-; The epilogue loop gets vectorised vscale x 2 x i16 wide.
+; The epilogue loop gets vectorised 4 x i16 wide.
; CHECK-EPILOG-V2: vec.epilog.ph:
; CHECK-EPILOG-V2: vec.epilog.vector.body:
-; CHECK-EPILOG-V2: load <vscale x 2 x i16>
+; CHECK-EPILOG-V2: load <4 x i16>
+
+; The epilogue loop gets vectorised vscale x 2 x i16 wide.
+; CHECK-EPILOG-V2-PREF-SCALBLE: vec.epilog.ph:
+; CHECK-EPILOG-V2-PREF-SCALBLE: vec.epilog.vector.body:
+; CHECK-EPILOG-V2-PREF-SCALBLE: load <vscale x 2 x i16>
; CHECK-NO-EPILOG-NOT: vec.epilog.vector.ph:
; CHECK-NO-EPILOG-NOT: vec.epilog.vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
index 503c05a22c2f3..28d5db0731ce8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
@@ -22,7 +22,7 @@ target triple = "aarch64-linux-gnu"
; DEBUG-EPILOG-PREFER-SCALABLE: LV: Checking a loop in 'main_vf_vscale_x_16'
; DEBUG-EPILOG-PREFER-SCALABLE: Create Skeleton for epilogue vectorized loop (first pass)
-; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:vscale x 16, Epilogue Loop UF:1
+; DEBUG-EPILOG-PREFER-SCALABLE: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:16, Epilogue Loop UF:1
define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-LABEL: @main_vf_vscale_x_16(
@@ -77,13 +77,10 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
;
; CHECK-EPILOG-PREFER-SCALABLE-LABEL: @main_vf_vscale_x_16(
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vector.main.loop.iter.check:
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 5
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -106,20 +103,18 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP11]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.ph:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 4
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP14]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 15
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.vector.body:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX4]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP15]], align 1
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], [[TMP14]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: store <16 x i8> splat (i8 1), ptr [[TMP15]], align 1
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 16
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 50942fc5adafb..f14e913c9e15a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -8,13 +8,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: iter.check:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.main.loop.iter.check:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
@@ -37,23 +34,26 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP25]]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i32>, ptr [[TMP18]], align 4
-; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD5]] to <vscale x 2 x i64>
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 2 x i64> [[TMP19]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> [[TMP20]], i32 1, <vscale x 2 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP25]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x i32>, ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = zext <2 x i32> [[WIDE_LOAD5]] to <2 x i64>
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i64> [[TMP16]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP16]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP9]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x ptr> [[TMP20]], ptr [[TMP19]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.v2p0.i32(<2 x ptr> [[TMP13]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
index e756076f27fd2..3d44e80bd9322 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -144,13 +144,10 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP12]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP11]], 5
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -204,38 +201,34 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP14]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP33:%.*]] = shl nuw i64 [[TMP27]], 3
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP0]], [[TMP33]]
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = and i64 [[TMP0]], 7
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF10]]
; CHECK-NEXT: [[TMP32:%.*]] = sub i64 [[N]], [[N_VEC11]]
-; CHECK-NEXT: [[TMP37:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT18:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <vscale x 4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <8 x i32> [ [[TMP26]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX12]]
; CHECK-NEXT: [[TMP34:%.*]] = load i16, ptr [[SRC]], align 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP34]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT14]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP40:%.*]] = sext <vscale x 8 x i16> [[BROADCAST_SPLAT15]] to <vscale x 8 x i32>
-; CHECK-NEXT: [[PARTIAL_REDUCE16]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv8i32(<vscale x 4 x i32> [[VEC_PHI13]], <vscale x 8 x i32> [[TMP40]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <8 x i16> poison, i16 [[TMP34]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT14]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = sext <8 x i16> [[BROADCAST_SPLAT15]] to <8 x i32>
+; CHECK-NEXT: [[TMP33]] = add <8 x i32> [[VEC_PHI13]], [[TMP27]]
; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP41:%.*]] = sub nuw nsw i64 [[TMP33]], 1
-; CHECK-NEXT: [[TMP35:%.*]] = sub i64 0, [[TMP41]]
-; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i16, ptr [[TMP36]], i64 [[TMP35]]
-; CHECK-NEXT: [[REVERSE17:%.*]] = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> [[BROADCAST_SPLAT15]])
-; CHECK-NEXT: store <vscale x 8 x i16> [[REVERSE17]], ptr [[TMP42]], align 2
-; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], [[TMP33]]
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i16, ptr [[TMP36]], i64 -7
+; CHECK-NEXT: [[REVERSE16:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLAT15]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: store <8 x i16> [[REVERSE16]], ptr [[TMP35]], align 2
+; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX12]], 8
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT18]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[TMP39]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-NEXT: [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP33]])
; CHECK-NEXT: [[CMP_N19:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC11]]
; CHECK-NEXT: br i1 [[CMP_N19]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
More information about the llvm-commits
mailing list