[llvm] [LoopVectorize] Improve Vectorization of Low Trip Count Loops (PR #195823)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 05:56:16 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/195823
>From af31834f52201e271f7e3259fc46835d6652abff Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 1 May 2026 11:43:47 +0100
Subject: [PATCH 01/25] [LoopVectorize] Improve Vectorization of Small Loops
Currently, Small Loops with Trip Counts less than 16, and in
situations where the Trip Count (TC) is less than the Tail Folding
Threshold are harder to vectorize, its only possible where no epilogue
will be emitted. However, for loops with large bodies and small trip
counts this can be counterprodictive to performance, often failing to
vectorize entirely. This is more prevelant with targets where
`getMinTripCountTailFoldingThreshold()` returns a value greater than 0.
To address this, the Small Loops where the TC == VF + 1 can now vectorize,
leading to a single vectorized itneration and a single scalar iteration.
Later passes can then remove the loop's entirely.
Testing an with OpenSource Fortran HPC Benchmark which includes multiple
loops with small trip counts, but large loop bodies, has shown significant
improvement to runtime after these changes.
Assisted-by: Claude Sonnet 4.6/Codex
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++-
.../AArch64/sve-low-trip-count.ll | 24 +--
.../sve-small-trip-count-vf-plus-one.ll | 195 ++++++++++++++++++
3 files changed, 243 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3eb6d0530da6d..a53597992d87f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3068,6 +3068,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
+ auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
+ unsigned int FixedVF, unsigned int ScalableVF) {
+ MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
+ };
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3079,9 +3088,36 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- MaxFactors.ScalableVF = ElementCount::getScalable(0);
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
return MaxFactors;
}
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // 2 between 2 and MaxVF.
+ //
+ // This produces 1 vector iteration, and 1 scalar iteration with
+ // no remainder. Later passes will eliminate the loop and leave
+ // straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
+ ExactTC && ExactTC.isFixed()) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
+ "iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ return MaxFactors;
+ }
+ // If the maximum VF cannot produce 1 vector iteration + 1 scalar
+ // iteration, step down VF's to find one that can.
+ for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
+ VF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ << " with 1 scalar iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, VF, 0);
+ return MaxFactors;
+ }
+ }
+ }
}
reportVectorizationFailure(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index c36daa40f6193..76642c63dbd26 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -56,22 +56,20 @@ exit:
define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
; CHECK-LABEL: define void @trip5_i8(
; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
-; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
-; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..71157580334d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
+; produces one vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
+; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
+define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_vf2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=4: exact multiple of VF=4. Vectorizes via the original
+; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
+define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc4_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 4
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From acfb67acef0916f3e6a475aa65df7541476f0e9d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 11:29:55 +0100
Subject: [PATCH 02/25] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a53597992d87f..833fa34a79aac 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3074,7 +3074,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
};
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
+ unsigned int VF) -> bool {
return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3100,7 +3101,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ if (HasOneScalarIterationRemainder(
+ ExactTC, MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
"iteration remainder.\n");
ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
>From 4189d9019281313f3c1fe3e75316c8bc7269008a Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 15:50:14 +0100
Subject: [PATCH 03/25] Responding to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++---
.../AArch64/sve-low-trip-count.ll | 38 ------
.../sve-small-trip-count-vf-plus-one.ll | 86 +-------------
.../RISCV/small-trip-count-vf-plus-one.ll | 109 ++++++++++++++++++
4 files changed, 127 insertions(+), 144 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 833fa34a79aac..193d3d657e4d3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3068,15 +3068,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
- unsigned int FixedVF, unsigned int ScalableVF) {
- MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
- MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
- };
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int VF) -> bool {
- return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ unsigned int MaxVF) -> bool {
+ return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
@@ -3089,7 +3084,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
// Allow cases where the ExactTC == VF + 1. VF can be any power of
@@ -3100,22 +3095,21 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(
- ExactTC, MaxFactors.FixedVF.getFixedValue())) {
- LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
- "iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
- return MaxFactors;
- }
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can.
- for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
- VF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ // iteration, step down VF's to find one that can. The result should
+ // also eliminate any loops.
+ //
+ // Forced interleaving is considered when seeing if OneScalarIterationRemainder
+ // is produced. It may prodiced more than one vector iteration, but only one
+ // scalar iteration.
+ for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
+ MaxVF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, VF, 0);
+ MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index 76642c63dbd26..03746ee82223a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -53,42 +53,4 @@ exit:
ret void
}
-define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
-; CHECK-LABEL: define void @trip5_i8(
-; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
-; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
- %0 = load i8, ptr %gep.src, align 1
- %mul = shl i8 %0, 1
- %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
- %1 = load i8, ptr %gep.dst, align 1
- %add = add i8 %mul, %1
- store i8 %add, ptr %gep.dst, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 5
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 71157580334d1..711ce3bc439d2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
-; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
-; produces one vector iteration and one scalar iteration.
+; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
;
; RUN: opt -S -p loop-vectorize %s | FileCheck %s
@@ -107,88 +107,6 @@ exit:
ret void
}
-; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
-; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
-define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc3_vf2_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
-; TC=4: exact multiple of VF=4. Vectorizes via the original
-; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
-define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc4_vf4_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 4
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..39917f15e2870
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on RISCV where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s -mtriple=riscv64 -mattr=+v -tail-folding-policy=dont-fold-tail | FileCheck %s
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From 220a85b4c6e329e8013b2f0b9c0fc64fbbad1f01 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 16:30:20 +0100
Subject: [PATCH 04/25] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 193d3d657e4d3..5efc4c3132118 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3069,8 +3069,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int MaxVF) -> bool {
+ auto HasOneScalarIterationRemainder =
+ [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3099,10 +3099,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if OneScalarIterationRemainder
- // is produced. It may prodiced more than one vector iteration, but only one
- // scalar iteration.
+ //
+ // Forced interleaving is considered when seeing if
+ // OneScalarIterationRemainder is produced. It may prodiced more than
+ // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
>From aa707f85d0c3d54a2f00a48ce7346807907ea98b Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 6 May 2026 08:56:45 +0100
Subject: [PATCH 05/25] Use ExpectedTC
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5efc4c3132118..4830488e0234d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3087,15 +3087,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExpectedTC->isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3105,7 +3104,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
>From 9603065cbcdc1ac6aca58d5ec13fff73591c20b1 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 10:12:06 +0100
Subject: [PATCH 06/25] Update OPT Test and readd ExactTc
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../sve-small-trip-count-vf-plus-one.ll | 184 ++++++++++++++++--
2 files changed, 170 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4830488e0234d..5efc4c3132118 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3087,14 +3087,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExpectedTC->isFixed()) {
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3104,7 +3105,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 711ce3bc439d2..7e93ae7f518b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -13,9 +13,9 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
@@ -23,11 +23,11 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -35,7 +35,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -55,14 +55,110 @@ exit:
ret void
}
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; the low-trip-count path steps down to a smaller profitable VF.
+define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[A]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i16> [[WIDE_LOAD]], splat (i16 1)
+; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[B]], align 2
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-NEXT: [[ADD:%.*]] = add i16 [[VAL]], 1
+; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_B]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i16, ptr %b, i64 %iv
+ %val = load i16, ptr %gep.a, align 2
+ %add = add i16 %val, 1
+ store i16 %add, ptr %gep.b, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; search can step down more than once before accepting VF=4.
+define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[VAL]], 1
+; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %val = load i8, ptr %gep.a, align 1
+ %add = add i8 %val, 1
+ store i8 %add, ptr %gep.b, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
@@ -75,11 +171,11 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -87,7 +183,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -107,6 +203,58 @@ exit:
ret void
}
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i64(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[WIDE_LOAD1]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[VAL]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %val = load i64, ptr %gep.a, align 4
+ %add = add nsw i64 %val, 1
+ store i64 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 2d34f966a88f89354aed0d9856640e58f5b6359b Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 14:42:25 +0100
Subject: [PATCH 07/25] Check for ExactTC!=0
The checks for matching to VF+1 == TC should protect against this, but
its best to the explicit.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 53 +++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5efc4c3132118..90a9eb4cfc907 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3095,7 +3095,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExactTC.getFixedValue() != 0) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 7e93ae7f518b3..4b80ffb894817 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -255,6 +255,59 @@ exit:
ret void
}
+; ExactTC is unknown here because the loop trip count is the runtime value %n,
+; but the guard proves the maximum trip count is 5. This should still take the
+; low-trip-count path, but it must not use the VF+1 escape because
+; getSmallConstantTripCount returns 0.
+define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
+; CHECK-LABEL: define void @unknown_exact_tc_max5(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IS_ZERO:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[IS_ZERO]], label %[[EXIT:.*]], label %[[GUARD:.*]]
+; CHECK: [[GUARD]]:
+; CHECK-NEXT: [[TOO_LARGE:%.*]] = icmp ugt i64 [[N]], 5
+; CHECK-NEXT: br i1 [[TOO_LARGE]], label %[[EXIT]], label %[[LOOP_PREHEADER:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %is.zero = icmp eq i64 %n, 0
+ br i1 %is.zero, label %exit, label %guard
+
+guard:
+ %too.large = icmp ugt i64 %n, 5
+ br i1 %too.large, label %exit, label %loop
+
+loop:
+ %iv = phi i64 [ 0, %guard ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 452e9d025053edb640d83da7ddd4035589cf89ba Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 19 May 2026 11:23:52 +0100
Subject: [PATCH 08/25] Respond to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 88 +++++++++++++++++--
2 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 90a9eb4cfc907..e762fe0cadcf8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3101,7 +3101,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// also eliminate any loops.
//
// Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may prodiced more than
+ // OneScalarIterationRemainder is produced. It may produced more than
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 4b80ffb894817..d9db7d31d3b87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,17 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
; produces the required number of vector iteration and one scalar iteration.
-;
-; RUN: opt -S -p loop-vectorize %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
-define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_vf4_vectorize(
+define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -41,6 +40,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -51,6 +51,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -89,6 +90,7 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
@@ -99,6 +101,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -137,6 +140,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
@@ -147,14 +151,15 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
-define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -189,6 +194,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -199,6 +205,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
@@ -241,6 +248,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -251,14 +259,14 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. This should still take the
-; low-trip-count path, but it must not use the VF+1 escape because
-; getSmallConstantTripCount returns 0.
+; but the guard proves the maximum trip count is 5. it must not use the VF+1
+; escape because getSmallConstantTripCount returns 0.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -308,7 +316,69 @@ exit:
ret void
}
+; TC=5, VF=4, UserIC=4
+; The user interleave count should be ignored because the dependence distance
+; makes the loop unsafe for interleaving > 1. Vectorization should still pick
+; VF=4 and produce 1 vector iteration plus 1 scalar iteration.
+define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_unsafe_useric_distance4_i32(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP0]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B_DST:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[B_SRC:%.*]] = getelementptr inbounds i32, ptr [[B_DST]], i64 -4
+; CHECK-NEXT: [[DEP:%.*]] = load i32, ptr [[B_SRC]], align 4
+; CHECK-NEXT: [[A_SRC:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[A_SRC]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], [[DEP]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %b.dst = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b.src = getelementptr inbounds i32, ptr %b.dst, i64 -4
+ %dep = load i32, ptr %b.src, align 4
+ %a.src = getelementptr inbounds i32, ptr %a, i64 %iv
+ %val = load i32, ptr %a.src, align 4
+ %add = add nsw i32 %val, %dep
+ store i32 %add, ptr %b.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 9
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = distinct !{!2, !3, !4}
+!3 = !{!"llvm.loop.interleave.count", i32 4}
+!4 = !{!"llvm.loop.vectorize.width", i32 4}
>From efc67e16166f603e9a46480b670b7e272ac63054 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:23:41 +0000
Subject: [PATCH 09/25] Reuse EffectiveIC in NoScalarEpilogueNeeded
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e762fe0cadcf8..37e020226c458 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3034,13 +3034,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
- auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
+ auto NoScalarEpilogueNeeded = [this](unsigned MaxVF, unsigned EffectiveIC) {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
!Legal->hasUncountableEarlyExit())
return false;
- unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
+ unsigned MaxVFtimesIC = MaxVF * EffectiveIC;
ScalarEvolution *SE = PSE.getSE();
// Calling getSymbolicMaxBackedgeTakenCount enables support for loops
// with uncountable exits. For countable loops, the symbolic maximum must
@@ -3057,10 +3057,11 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return Rem->isZero();
};
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
if (MaxPowerOf2RuntimeVF > 0u) {
assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
"MaxFixedVF must be a power of 2");
- if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
+ if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF, EffectiveIC)) {
// Accept MaxFixedVF if we do not have a tail.
LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
return MaxFactors;
@@ -3068,7 +3069,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder =
[EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
@@ -3081,7 +3081,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 0c3fabb75a3da0a0bf45769f4380c541af535633 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:51:36 +0000
Subject: [PATCH 10/25] format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 37e020226c458..0699ea41d387e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3081,7 +3081,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(),
+ EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 72739f513be9a634b0c0aab01f04af30ac652d0f Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 28 May 2026 11:51:40 +0100
Subject: [PATCH 11/25] Address nit comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++-----
.../AArch64/sve-small-trip-count-vf-plus-one.ll | 10 +++++-----
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0699ea41d387e..d8d71aeabef5c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3100,15 +3100,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may produced more than
- // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
+ // OneScalarIterationRemainder takes account of any forced
+ // interleaving.
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
- << " with 1 scalar iteration remainder.\n");
+ << " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index d9db7d31d3b87..5c08f97a55aa2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -57,7 +57,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; VF=8 is a natural fixed-width for i16 types on AArch64, so this also checks that
; the low-trip-count path steps down to a smaller profitable VF.
define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
@@ -107,7 +107,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; VF=16 is a natural fixed-width for i8 types on AArch64, so this checks that the
; search can step down more than once before accepting VF=4.
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
@@ -264,9 +264,9 @@ exit:
ret void
}
-; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. it must not use the VF+1
-; escape because getSmallConstantTripCount returns 0.
+; In this case the vectoriser shouldn't optimise for a single vector iteration
+; + single scalar iteration, because there is no guarantee we will enter the vector
+; loop.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
>From a9b5f2563aacdc821ca6a5c18d48edad4b7c0e71 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 09:54:07 +0100
Subject: [PATCH 12/25] Add cost modelling for where scalar loops are more
profitable
---
.../Vectorize/LoopVectorizationPlanner.cpp | 42 +++---
.../Vectorize/LoopVectorizationPlanner.h | 5 +
.../Transforms/Vectorize/LoopVectorize.cpp | 77 ++++++++++
.../sve-small-trip-count-vf-plus-one-cost.ll | 132 ++++++++++++++++++
4 files changed, 237 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index fd7fd2e011a83..8507c60a2a94d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -738,25 +738,8 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
if (!MaxTripCount)
return LowerCostWithoutTC;
- auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
- InstructionCost VectorCost,
- InstructionCost ScalarCost) {
- // If the trip count is a known (possibly small) constant, the trip count
- // will be rounded up to an integer number of iterations under
- // FoldTailByMasking. The total cost in that case will be
- // VecCost*ceil(TripCount/VF). When not folding the tail, the total
- // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
- // some extra overheads, but for the purpose of comparing the costs of
- // different VFs we can use this to compare the total loop-body cost
- // expected after vectorization.
- if (HasTail)
- return VectorCost * (MaxTripCount / VF) +
- ScalarCost * (MaxTripCount % VF);
- return VectorCost * divideCeil(MaxTripCount, VF);
- };
-
- auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
- auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
+ auto RTCostA = getCostForKnownTripCount(A, MaxTripCount, HasTail);
+ auto RTCostB = getCostForKnownTripCount(B, MaxTripCount, HasTail);
bool LowerCostWithTC = CmpFn(RTCostA, RTCostB);
LLVM_DEBUG(if (LowerCostWithTC != LowerCostWithoutTC) {
dbgs() << "LV: VF " << (LowerCostWithTC ? A.Width : B.Width)
@@ -778,6 +761,27 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
IsEpilogue);
}
+InstructionCost LoopVectorizationPlanner::getCostForKnownTripCount(
+ const VectorizationFactor &VF, unsigned TripCount, bool HasTail) const {
+ unsigned EstimatedWidth = VF.Width.getKnownMinValue();
+ if (std::optional<unsigned> VScale = Config.getVScaleForTuning())
+ if (VF.Width.isScalable())
+ EstimatedWidth *= *VScale;
+
+ // If the trip count is a known (possibly small) constant, the trip count
+ // will be rounded up to an integer number of iterations under
+ // FoldTailByMasking. The total cost in that case will be
+ // VecCost*ceil(TripCount/VF). When not folding the tail, the total
+ // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
+ // some extra overheads, but for the purpose of comparing the costs of
+ // different VFs we can use this to compare the total loop-body cost
+ // expected after vectorization.
+ if (HasTail)
+ return VF.Cost * (TripCount / EstimatedWidth) +
+ VF.ScalarCost * (TripCount % EstimatedWidth);
+ return VF.Cost * divideCeil(TripCount, EstimatedWidth);
+}
+
// TODO: we could return a pair of values that specify the max VF and
// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fa317f290022e..ba6ca02c5d562 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -1041,6 +1041,11 @@ class LoopVectorizationPlanner {
const unsigned MaxTripCount, bool HasTail,
bool IsEpilogue = false) const;
+ /// Returns the estimated loop-body cost for \p VF and a known trip count.
+ InstructionCost getCostForKnownTripCount(const VectorizationFactor &VF,
+ unsigned TripCount,
+ bool HasTail) const;
+
/// Determines if we have the infrastructure to vectorize the loop and its
/// epilogue, assuming the main loop is vectorized by \p MainPlan.
bool isCandidateForEpilogueVectorization(VPlan &MainPlan) const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d8d71aeabef5c..ca2006314572a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5868,6 +5868,46 @@ LoopVectorizationPlanner::computeBestVF() {
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
+ bool ForceVectorization =
+ Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
+ hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned EstimatedWidth =
+ estimateElementCount(UserVF, Config.getVScaleForTuning());
+ if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
+ TC == EstimatedWidth + 1) {
+ ElementCount ScalarVF = ElementCount::getFixed(1);
+ InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ LLVM_DEBUG(dbgs()
+ << "LV: Scalar loop costs: " << ScalarCost << ".\n");
+
+ InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
+ VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
+ VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces
+ // extra control flow and a scalar epilogue for a single element, so
+ // require the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
+ AdjustedVectorCost >= ScalarCostForTC) {
+ LLVM_DEBUG(dbgs()
+ << "LV: Rejecting VF " << UserVF
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return {ScalarFactor, &FirstPlan};
+ }
+ }
+ }
+ }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -5909,6 +5949,40 @@ LoopVectorizationPlanner::computeBestVF() {
}
VPlan *PlanForBestVF = &FirstPlan;
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC % EstimatedWidth != 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -5945,6 +6019,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ continue;
+
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
BestFactor = CurrentFactor;
PlanForBestVF = P.get();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
new file mode 100644
index 0000000000000..82cbb431ed76f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -0,0 +1,132 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -S %s | FileCheck %s --check-prefix=IR
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s --check-prefix=DBG
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body:
+;
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
+; DBG: LV: Using user VF 2.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 10.
+; DBG: Cost for VF 2: 15
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
+; DBG-NOT: Rejecting VF 2
+; DBG: LV: Selecting VF: 2.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+declare i8 @llvm.smin.i8(i8, i8)
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.width", i32 2}
>From d83fc0745ebe0851fbd26ef1c4da2df1fdeff5a6 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 14:06:50 +0100
Subject: [PATCH 13/25] Remove for loop for calculating best VF
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 +++++++------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ca2006314572a..8818628e8079e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3069,10 +3069,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto HasOneScalarIterationRemainder =
- [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
- return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
- };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3096,18 +3092,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.getFixedValue() != 0) {
- // If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can. The result should
- // also eliminate any loops.
- for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
- MaxVF /= 2) {
- // OneScalarIterationRemainder takes account of any forced
- // interleaving.
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
+ ExactTC.getFixedValue() > 1) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned MaxFixedVF = MaxFactors.FixedVF.getFixedValue();
+ if ((TC - 1) % EffectiveIC == 0) {
+ unsigned VF = (TC - 1) / EffectiveIC;
+ if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
<< " with 1 scalar iteration remaining.\n");
- MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
@@ -5962,7 +5955,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC % EstimatedWidth != 1)
+ if (TC != EstimatedWidth + 1)
return false;
InstructionCost VectorCost =
>From bd2442ba5d4f2af3aa81807fbbe41492fb06e983 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 23 Jun 2026 09:30:56 +0100
Subject: [PATCH 14/25] Refactor IsUnprofitableOneScalarTail Lambda Function
---
.../Transforms/Vectorize/LoopVectorize.cpp | 92 +++++++++----------
1 file changed, 42 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8818628e8079e..d60c96bd8fb49 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5853,6 +5853,42 @@ LoopVectorizationPlanner::computeBestVF() {
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ const InstructionCost &ScalarCost) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != EstimatedWidth + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!AdjustedVectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
ElementCount UserVF = Hints.getWidth();
if (VPlans.size() == 1) {
@@ -5879,23 +5915,10 @@ LoopVectorizationPlanner::computeBestVF() {
InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces
- // extra control flow and a scalar epilogue for a single element, so
- // require the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
- AdjustedVectorCost >= ScalarCostForTC) {
- LLVM_DEBUG(dbgs()
- << "LV: Rejecting VF " << UserVF
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
+ if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
+ ForceVectorization, ExactTC,
+ ScalarFactor, ScalarCost)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -5943,39 +5966,6 @@ LoopVectorizationPlanner::computeBestVF() {
VPlan *PlanForBestVF = &FirstPlan;
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -6012,7 +6002,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor,
+ ScalarCost))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
>From b8189455864759e415a3fc3ba349d946b7e6673e Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 2 Jul 2026 15:54:45 +0100
Subject: [PATCH 15/25] Add consideration of IC and test
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 ++--
.../sve-small-trip-count-vf-plus-one-cost.ll | 46 ++++++++++++++++---
2 files changed, 44 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d60c96bd8fb49..0b26947692ee7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5857,7 +5857,7 @@ LoopVectorizationPlanner::computeBestVF() {
[&](const VectorizationFactor &CurrentFactor, bool HasTail,
bool ForceVectorization, const ElementCount &ExactTC,
const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost) {
+ const InstructionCost &ScalarCost, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5868,7 +5868,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
+ if (TC != (EstimatedWidth * UserIC) + 1)
return false;
InstructionCost VectorCost =
@@ -5891,6 +5891,7 @@ LoopVectorizationPlanner::computeBestVF() {
};
ElementCount UserVF = Hints.getWidth();
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
@@ -5918,7 +5919,7 @@ LoopVectorizationPlanner::computeBestVF() {
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost)) {
+ ScalarFactor, ScalarCost, UserIC)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -6004,7 +6005,7 @@ LoopVectorizationPlanner::computeBestVF() {
if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost))
+ ScalarCost, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 82cbb431ed76f..70eb5b07c25aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -7,13 +7,7 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc3_udiv_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_forced(
-; IR: vector.body:
+; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
@@ -44,6 +38,9 @@ exit:
define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
; DBG: LV: Using user VF 2.
; DBG: LV: Scalar loop costs: 9.
@@ -71,6 +68,9 @@ exit:
}
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
@@ -99,6 +99,9 @@ exit:
define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
; DBG-NOT: Rejecting VF 2
; DBG: LV: Selecting VF: 2.
@@ -122,6 +125,33 @@ exit:
ret void
}
+define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0{
+; IR-LABEL: define void @tc5_udiv_i8_reject_ic2(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
+; DBG-NOT: LV: Selecting VF: 2.
+; DBG Rejecting VF 2
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+
+exit:
+ ret void
+}
+
declare i8 @llvm.smin.i8(i8, i8)
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
@@ -130,3 +160,5 @@ attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.vectorize.width", i32 2}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.interleave.count", i32 2}
>From e1aa15fce90289dc17c174b598dbb65541f39531 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:25:20 +0100
Subject: [PATCH 16/25] Respond to review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 7 ++
.../Transforms/Vectorize/LoopVectorize.cpp | 109 +++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 75 ++++++------
.../sve-small-trip-count-vf-plus-one.ll | 81 +++++++++++++
4 files changed, 166 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ba6ca02c5d562..5c1ffe93b90c6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -911,6 +911,13 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
+ /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
+ /// it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0b26947692ee7..06a5c98c1fb11 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3098,7 +3098,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if ((TC - 1) % EffectiveIC == 0) {
unsigned VF = (TC - 1) / EffectiveIC;
if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
<< " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
@@ -5847,84 +5847,57 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != (EstimatedWidth * UserIC) + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
+ if (!VectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+}
+
std::pair<VectorizationFactor, VPlan *>
LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost, unsigned int UserIC) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != (EstimatedWidth * UserIC) + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!AdjustedVectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
ElementCount UserVF = Hints.getWidth();
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
- bool ForceVectorization =
- Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
- hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
- unsigned TC = ExactTC.getFixedValue();
- unsigned EstimatedWidth =
- estimateElementCount(UserVF, Config.getVScaleForTuning());
- if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
- TC == EstimatedWidth + 1) {
- ElementCount ScalarVF = ElementCount::getFixed(1);
- InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
- LLVM_DEBUG(dbgs()
- << "LV: Scalar loop costs: " << ScalarCost << ".\n");
-
- InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
- VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
- VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
- ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost, UserIC)) {
- return {ScalarFactor, &FirstPlan};
- }
- }
- }
- }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -6003,9 +5976,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost, UserIC))
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 70eb5b07c25aa..4516062f9b7d1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -10,7 +10,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
@@ -36,43 +36,12 @@ exit:
ret void
}
-define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
- ptr noalias %c) #0 {
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-
-; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
-; DBG: LV: Using user VF 2.
-; DBG: LV: Scalar loop costs: 9.
-; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
-; DBG: LV: Vectorization is possible but not beneficial.
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %pa = getelementptr inbounds i8, ptr %a, i64 %iv
- %pb = getelementptr inbounds i8, ptr %b, i64 %iv
- %pc = getelementptr inbounds i8, ptr %c, i64 %iv
- %va = load i8, ptr %pa, align 1
- %vb = load i8, ptr %pb, align 1
- %div = udiv i8 %va, %vb
- store i8 %div, ptr %pc, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
-
-exit:
- ret void
-}
-
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
; IR-NOT: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
@@ -131,7 +100,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG Rejecting VF 2
+; DBG: Rejecting VF 2
entry:
br label %loop
@@ -146,7 +115,37 @@ loop:
store i8 %div, ptr %pc, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc5_sin_f32_select_smaller_vf(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
+; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
+; DBG: LV: Rejecting VF 4 for one-scalar-tail low trip count: vector cost 88 >= scalar cost 80.
+; DBG-NOT: Selecting VF: 4
+; DBG: LV: Selecting VF: 2.
+
+
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds float, ptr %a, i64 %iv
+ %pc = getelementptr inbounds float, ptr %c, i64 %iv
+ %va = load float, ptr %pa, align 4
+ %sin = tail call float @llvm.sin.f32(float %va)
+ store float %sin, ptr %pc, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
exit:
ret void
@@ -154,11 +153,11 @@ exit:
declare i8 @llvm.smin.i8(i8, i8)
+declare float @llvm.sin.f32(float)
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
-!3 = !{!"llvm.loop.vectorize.width", i32 2}
-!4 = distinct !{!4, !5}
-!5 = !{!"llvm.loop.interleave.count", i32 2}
+!3 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 5c08f97a55aa2..b999acc21309c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -375,6 +375,87 @@ exit:
ret void
}
+; For this example, vectorization should not take place as the
+; scalar cost is better than the vectorized cost.
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+; CHECK-LABEL: define void @tc3_udiv_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[PB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[PC:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT: [[VA:%.*]] = load i8, ptr [[PA]], align 1
+; CHECK-NEXT: [[VB:%.*]] = load i8, ptr [[PB]], align 1
+; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[VA]], [[VB]]
+; CHECK-NEXT: store i8 [[DIV]], ptr [[PC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+ ptr noalias %c) #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_smin_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[TMP1]], i8 [[TMP2]])
+; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 00e6801574c0f61cbf93e496bb688bec7662213c Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:55:34 +0100
Subject: [PATCH 17/25] format
---
.../Vectorize/LoopVectorizationPlanner.h | 14 +++++----
.../Transforms/Vectorize/LoopVectorize.cpp | 29 +++++++++++--------
2 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 5c1ffe93b90c6..8c43b22217860 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -911,12 +911,14 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
- /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
- /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
- /// it is better to leave the original Scalar loop in place.
- bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that
+ /// leaves a Vector iteration, followed by a single iteration scalar tail. For
+ /// some uses cases, it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ bool HasTail, bool ForceVectorization,
+ const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 06a5c98c1fb11..b2f265eb8623e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5847,9 +5847,10 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5858,8 +5859,8 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
return false;
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
+ unsigned EstimatedWidth =
+ estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
if (TC != (EstimatedWidth * UserIC) + 1)
return false;
@@ -5871,11 +5872,13 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
// control flow and a scalar epilogue for a single element, so require
// the vectorized form to save at least one scalar iteration.
InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
- if (!VectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC) {
- LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
- return false;
- }
+ if (!VectorCost.isValid() || AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " < " << ScalarCostForTC
+ << ".\n");
+ return false;
+ }
LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
@@ -5976,9 +5979,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ unsigned int UserIC =
+ Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor, UserIC))
+ ForceVectorization, ExactTC, ScalarFactor,
+ UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
>From 10975f85a1664390e647b495899faf32c198c042 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 20 Jul 2026 10:04:12 +0100
Subject: [PATCH 18/25] Respond to review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 13 +++---
.../Transforms/Vectorize/LoopVectorize.cpp | 42 ++++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 8 ++--
.../sve-small-trip-count-vf-plus-one.ll | 24 ++++++-----
4 files changed, 42 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 8c43b22217860..11573885e6f1a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -912,13 +912,12 @@ class LoopVectorizationPlanner {
VPlan &getPlanFor(ElementCount VF) const;
/// Examines if it is unprofitable to Vectorize a small loop in a way that
- /// leaves a Vector iteration, followed by a single iteration scalar tail. For
- /// some uses cases, it is better to leave the original Scalar loop in place.
- bool isUnprofitableOneScalarTail(const VectorizationFactor &CurrentFactor,
- bool HasTail, bool ForceVectorization,
- const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor,
- unsigned int UserIC);
+ /// leaves a Vector iteration, followed by a single iteration scalar tail.
+ /// Returns true if it is profitable to vectorize a small loop with a single
+ /// scalar tail iteration
+ bool isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ const ElementCount &ExactTC,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b2f265eb8623e..ec889b4fd47b7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5847,44 +5847,39 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isUnprofitableOneScalarTail(
- const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
+bool LoopVectorizationPlanner::isProfitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, const ElementCount &ExactTC,
+ unsigned int UserIC) {
+ if (!ExactTC.isFixed() || CurrentFactor.Width.isScalable())
+ return true;
unsigned TC = ExactTC.getFixedValue();
if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
+ return true;
unsigned EstimatedWidth =
estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
if (TC != (EstimatedWidth * UserIC) + 1)
- return false;
+ return true;
+ // VectorCost reflects the cost of the requried vector iteration(s) and the
+ // one remaining scalar iteration cost
InstructionCost VectorCost =
getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
- if (!VectorCost.isValid() || AdjustedVectorCost < ScalarCostForTC) {
+ InstructionCost ScalarCostForTC = CurrentFactor.ScalarCost * TC;
+ if (!VectorCost.isValid() || VectorCost < ScalarCostForTC) {
LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " < " << ScalarCostForTC
+ << VectorCost << " < scalar cost " << ScalarCostForTC
<< ".\n");
- return false;
+ return true;
}
LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
+ << VectorCost << " >= scalar cost " << ScalarCostForTC
+ << ".\n");
+ return false;
}
std::pair<VectorizationFactor, VPlan *>
@@ -5981,9 +5976,8 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned int UserIC =
Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
- if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor,
- UserIC))
+ if (!ForceVectorization && P->hasScalarTail() &&
+ !isProfitableOneScalarTail(CurrentFactor, ExactTC, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 4516062f9b7d1..e877af9dc88e8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -13,7 +13,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 27.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -44,7 +44,7 @@ define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 40 >= scalar cost 30.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -100,7 +100,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG: Rejecting VF 2
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 85 >= scalar cost 45.
entry:
br label %loop
@@ -128,7 +128,7 @@ define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
-; DBG: LV: Rejecting VF 4 for one-scalar-tail low trip count: vector cost 88 >= scalar cost 80.
+; DBG: LV: Accepting VF 4 for one-scalar-tail low trip count: vector cost 72 < scalar cost 80.
; DBG-NOT: Selecting VF: 4
; DBG: LV: Selecting VF: 2.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index b999acc21309c..dc947ff778280 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -106,9 +106,8 @@ exit:
ret void
}
-; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
; VF=16 is a natural fixed-width for i8 types on AArch64, so this checks that the
-; search can step down more than once before accepting VF=4.
+; search can step down more than once before accepting the most acceptable VF.
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
@@ -117,10 +116,15 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -134,7 +138,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -188,7 +192,7 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -242,7 +246,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -351,7 +355,7 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 595b235901b080f7cf653e56a1fc98699b8c7122 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 21 Jul 2026 13:23:18 +0100
Subject: [PATCH 19/25] Fix test failures
---
.../sve-small-trip-count-vf-plus-one-cost.ll | 14 +-
.../sve-small-trip-count-vf-plus-one.ll | 142 ++++++++++--------
2 files changed, 84 insertions(+), 72 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index e877af9dc88e8..5c7da2b49df27 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -13,7 +13,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 27.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 28 >= scalar cost 27.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -36,17 +36,19 @@ exit:
ret void
}
+; FIXME: This is currently accepted as cost for vector is smaller than scalar.
+; Performance is poor due to poor CodeGen of using type promotion instead of
+; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
+; IR: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 40 >= scalar cost 30.
-; DBG: LV: Selecting VF: 1.
-; DBG: LV: Vectorization is possible but not beneficial.
+; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 25 < scalar cost 30.
+; DBG: LV: Selecting VF: 2.
entry:
br label %loop
@@ -100,7 +102,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 85 >= scalar cost 45.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 45.
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index dc947ff778280..f26f1fdc7ccce 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -12,9 +12,9 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH1:.*:]]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
@@ -22,11 +22,11 @@ define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -34,7 +34,7 @@ define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -62,9 +62,9 @@ exit:
define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[A]], align 2
@@ -72,11 +72,11 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[B]], align 2
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[GEP_A]], align 2
@@ -84,7 +84,7 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_B]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -111,26 +111,21 @@ exit:
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[GEP_A]], align 1
@@ -138,7 +133,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -165,9 +160,9 @@ exit:
define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH1:.*:]]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
@@ -180,11 +175,11 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -192,7 +187,7 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -219,9 +214,9 @@ exit:
define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i64(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
@@ -234,11 +229,11 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[GEP_A]], align 4
@@ -246,7 +241,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -327,9 +322,9 @@ exit:
define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_unsafe_useric_distance4_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 4
@@ -341,11 +336,11 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP0]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[B_DST:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[B_SRC:%.*]] = getelementptr inbounds i32, ptr [[B_DST]], i64 -4
; CHECK-NEXT: [[DEP:%.*]] = load i32, ptr [[B_SRC]], align 4
@@ -355,7 +350,7 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -422,13 +417,28 @@ exit:
ret void
}
+; FIXME: This is currently accepted as cost for vector is smaller than scalar.
+; Performance is poor due to poor CodeGen of using type promotion instead of
+; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc3_smin_i8_reject(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[B]], align 1
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[WIDE_LOAD]], <2 x i8> [[WIDE_LOAD1]])
+; CHECK-NEXT: store <2 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
@@ -437,7 +447,7 @@ define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 34eb2070b98e7d3e7ad2e8b641a89bfe527ff5bc Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 7 Aug 2026 09:28:15 +0100
Subject: [PATCH 20/25] Update test case name and fix failing tests
---
.../AArch64/sve-small-trip-count-vf-plus-one-cost.ll | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 5c7da2b49df27..d34b1d2585fff 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -36,14 +36,11 @@ exit:
ret void
}
-; FIXME: This is currently accepted as cost for vector is smaller than scalar.
-; Performance is poor due to poor CodeGen of using type promotion instead of
-; type widening.
-define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
-; IR-LABEL: define void @tc3_smin_i8_reject(
+define void @tc3_smin_i8_accept(ptr noalias %a, ptr noalias %b) #0 {
+; IR-LABEL: define void @tc3_smin_i8_accept(
; IR: vector.body
-; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
+; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_accept'
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
@@ -160,6 +157,6 @@ declare float @llvm.sin.f32(float)
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
-!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!1 = !{!"llvm.loop.vectorize.enable"}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.interleave.count", i32 2}
>From 940f0b4061753838431375e0c143238e5431779a Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 13 Aug 2026 15:45:20 +0100
Subject: [PATCH 21/25] Add minsize/optsize tests
---
.../sve-small-trip-count-vf-plus-one.ll | 97 ++++++++++++++++++-
1 file changed, 96 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index f26f1fdc7ccce..8771944dd9dc4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -8,7 +8,6 @@
target triple = "aarch64-unknown-linux-gnu"
; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
-; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -470,7 +469,103 @@ exit:
ret void
}
+define void @tc5_v4i32_reject_minsize(ptr noalias %a, ptr noalias %b) #1 {
+; CHECK-LABEL: define void @tc5_v4i32_reject_minsize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc5_v4i32_reject_optsize(ptr noalias %a, ptr noalias %b) #2 {
+; CHECK-LABEL: define void @tc5_v4i32_reject_optsize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+attributes #1 = { vscale_range(1,16) "target-features"="+sve" minsize }
+attributes #2 = { vscale_range(1,16) "target-features"="+sve" optsize }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
>From 5cc8ce366d4a04536a3584305e85ca5546f69121 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 13 Aug 2026 17:40:43 +0100
Subject: [PATCH 22/25] Update cost test after #212801
---
.../AArch64/sve-small-trip-count-vf-plus-one-cost.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index d34b1d2585fff..c374f163f807a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -43,8 +43,8 @@ define void @tc3_smin_i8_accept(ptr noalias %a, ptr noalias %b) #0 {
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_accept'
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
-; DBG: Cost for VF 2: 15
-; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 25 < scalar cost 30.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 29 < scalar cost 30.
; DBG: LV: Selecting VF: 2.
entry:
br label %loop
>From df65dc1b1610cda5b6404c7eb296ed94ecaae1f7 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 17 Aug 2026 10:53:38 +0100
Subject: [PATCH 23/25] Ensure transformation is not applied for
minsize/optsize
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +++-
.../sve-small-trip-count-vf-plus-one.ll | 34 ++++---------------
2 files changed, 11 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b9e88184f96a1..6396bb22a90a1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3077,9 +3077,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
+ //
+ // If a function is marked as minsize/optsize or OptForSize is set, do not
+ // allow this form of transformation as this will increase CodeSize.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.getFixedValue() > 1) {
+ ExactTC.getFixedValue() > 1 && !TheFunction->hasOptSize() &&
+ !Config.OptForSize) {
unsigned TC = ExactTC.getFixedValue();
unsigned MaxFixedVF = MaxFactors.FixedVF.getFixedValue();
if ((TC - 1) % EffectiveIC == 0) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 8771944dd9dc4..4d959504fb7fe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -472,21 +472,10 @@ exit:
define void @tc5_v4i32_reject_minsize(ptr noalias %a, ptr noalias %b) #1 {
; CHECK-LABEL: define void @tc5_v4i32_reject_minsize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -494,7 +483,7 @@ define void @tc5_v4i32_reject_minsize(ptr noalias %a, ptr noalias %b) #1 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -519,21 +508,10 @@ exit:
define void @tc5_v4i32_reject_optsize(ptr noalias %a, ptr noalias %b) #2 {
; CHECK-LABEL: define void @tc5_v4i32_reject_optsize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -541,7 +519,7 @@ define void @tc5_v4i32_reject_optsize(ptr noalias %a, ptr noalias %b) #2 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From e54c6cb60f0e59c96d052f0166e53af9ed368c27 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 18 Aug 2026 13:53:42 +0100
Subject: [PATCH 24/25] Respond to Review Comments
- Reuse GetLoopRemainder
- Add comments to tc5_sin_f32_select_smaller_vf to explain test
motivation
- Split isProfitableOneScalarTail into shouldConsiderOneScalarTail
and isProfitableOneScalarTail to make intention clearer
---
.../Vectorize/LoopVectorizationPlanner.h | 10 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 69 ++++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 4 ++
3 files changed, 42 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bd8da36d17355..feb711ef431f5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -911,13 +911,19 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
+ /// Returns true if, for the given loop, a One Iteration Scalar Tail should
+ /// be considered in cases where the trip count falls below the
+ /// tail folding threshold
+ bool shouldConsiderOneScalarTail(const VectorizationFactor &CurrentFactor,
+ const ElementCount &ExactTC,
+ unsigned int UserIC);
+
/// Examines if it is unprofitable to Vectorize a small loop in a way that
/// leaves a Vector iteration, followed by a single iteration scalar tail.
/// Returns true if it is profitable to vectorize a small loop with a single
/// scalar tail iteration
bool isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
- const ElementCount &ExactTC,
- unsigned int UserIC);
+ const ElementCount &ExactTC, unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6396bb22a90a1..c1644a4d43359 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3021,12 +3021,12 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
- auto NoScalarEpilogueNeeded = [this](unsigned MaxVF, unsigned EffectiveIC) {
+ auto GetLoopRemainder = [this](unsigned MaxVF, unsigned EffectiveIC)-> const SCEV * {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
!Legal->hasUncountableEarlyExit())
- return false;
+ return nullptr;
unsigned MaxVFtimesIC = MaxVF * EffectiveIC;
ScalarEvolution *SE = PSE.getSE();
// Calling getSymbolicMaxBackedgeTakenCount enables support for loops
@@ -3038,17 +3038,16 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
"Invalid loop count");
const SCEV *ExitCount = SE->getAddExpr(
BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
- const SCEV *Rem = SE->getURemExpr(
+ return SE->getURemExpr(
SE->applyLoopGuards(ExitCount, TheLoop),
SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
- return Rem->isZero();
};
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
if (MaxPowerOf2RuntimeVF > 0u) {
assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
"MaxFixedVF must be a power of 2");
- if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF, EffectiveIC)) {
+ if (const auto *Rem = GetLoopRemainder(*MaxPowerOf2RuntimeVF, EffectiveIC); Rem && Rem->isZero()) {
// Accept MaxFixedVF if we do not have a tail.
LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
return MaxFactors;
@@ -3059,20 +3058,17 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
- if (MaxPowerOf2RuntimeVF > 0u) {
+ if (MaxPowerOf2RuntimeVF > 0u && EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop) {
// If we have a low-trip-count, and the fixed-width VF is known to divide
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(),
- EffectiveIC)) {
+ if (const auto *Rem = GetLoopRemainder(MaxFactors.FixedVF.getFixedValue(), EffectiveIC); Rem && Rem->isZero()) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExactTC == VF + 1. VF can be any power of
- // 2 between 2 and MaxVF.
+ // Allow cases where the ExactTC == (VF * IC) + 1.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
@@ -3081,20 +3077,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If a function is marked as minsize/optsize or OptForSize is set, do not
// allow this form of transformation as this will increase CodeSize.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.getFixedValue() > 1 && !TheFunction->hasOptSize() &&
- !Config.OptForSize) {
+ if (ExactTC.getFixedValue() > 1 && !TheFunction->hasOptSize() && !Config.OptForSize) {
unsigned TC = ExactTC.getFixedValue();
- unsigned MaxFixedVF = MaxFactors.FixedVF.getFixedValue();
- if ((TC - 1) % EffectiveIC == 0) {
- unsigned VF = (TC - 1) / EffectiveIC;
- if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
- << " with 1 scalar iteration remaining.\n");
- MaxFactors.FixedVF = ElementCount::getFixed(VF);
- MaxFactors.ScalableVF = ElementCount::getScalable(0);
- return MaxFactors;
- }
+ unsigned VF = (TC - 1) / EffectiveIC;
+ if (const auto *Rem = GetLoopRemainder(VF, EffectiveIC); Rem && Rem->isOne() && isPowerOf2_32(VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
+ << " with 1 scalar iteration remaining.\n");
+ MaxFactors.FixedVF = ElementCount::getFixed(VF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
+ return MaxFactors;
}
}
}
@@ -5835,23 +5826,11 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isProfitableOneScalarTail(
- const VectorizationFactor &CurrentFactor, const ElementCount &ExactTC,
- unsigned int UserIC) {
- if (!ExactTC.isFixed() || CurrentFactor.Width.isScalable())
- return true;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return true;
-
- unsigned EstimatedWidth =
- estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != (EstimatedWidth * UserIC) + 1)
- return true;
-
+bool LoopVectorizationPlanner::isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ const ElementCount &ExactTC, unsigned int UserIC) {
// VectorCost reflects the cost of the requried vector iteration(s) and the
// one remaining scalar iteration cost
+ unsigned TC = ExactTC.getFixedValue();
InstructionCost VectorCost =
getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
InstructionCost ScalarCostForTC = CurrentFactor.ScalarCost * TC;
@@ -5870,6 +5849,17 @@ bool LoopVectorizationPlanner::isProfitableOneScalarTail(
return false;
}
+bool LoopVectorizationPlanner::shouldConsiderOneScalarTail(
+ const VectorizationFactor &CurrentFactor, const ElementCount &ExactTC,
+ unsigned int UserIC) {
+ if (ExactTC.isFixed()) {
+ unsigned TC = ExactTC.getFixedValue();
+ if (!CurrentFactor.Width.isScalable() && TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold())
+ return true;
+ }
+ return false;
+}
+
std::pair<VectorizationFactor, VPlan *>
LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
@@ -5965,6 +5955,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned int UserIC =
Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (!ForceVectorization && P->hasScalarTail() &&
+ shouldConsiderOneScalarTail(CurrentFactor, ExactTC, UserIC) &&
!isProfitableOneScalarTail(CurrentFactor, ExactTC, UserIC))
continue;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index c374f163f807a..335d6dba8c86d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -120,6 +120,9 @@ exit:
ret void
}
+; For examples where the MaxVF is less profitable than other
+; options, ensure the Loop Vectorizer can still pick the
+; most profitable option.
define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc5_sin_f32_select_smaller_vf(
@@ -127,6 +130,7 @@ define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
+; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 72 < scalar cost 80.
; DBG: LV: Accepting VF 4 for one-scalar-tail low trip count: vector cost 72 < scalar cost 80.
; DBG-NOT: Selecting VF: 4
; DBG: LV: Selecting VF: 2.
>From 2da9b20f36d5fab0507590ebf0053ca231669bb4 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 18 Aug 2026 13:55:44 +0100
Subject: [PATCH 25/25] formatting
---
.../Vectorize/LoopVectorizationPlanner.h | 7 ++---
.../Transforms/Vectorize/LoopVectorize.cpp | 27 ++++++++++++-------
2 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index feb711ef431f5..982321b219d3e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -915,15 +915,16 @@ class LoopVectorizationPlanner {
/// be considered in cases where the trip count falls below the
/// tail folding threshold
bool shouldConsiderOneScalarTail(const VectorizationFactor &CurrentFactor,
- const ElementCount &ExactTC,
- unsigned int UserIC);
+ const ElementCount &ExactTC,
+ unsigned int UserIC);
/// Examines if it is unprofitable to Vectorize a small loop in a way that
/// leaves a Vector iteration, followed by a single iteration scalar tail.
/// Returns true if it is profitable to vectorize a small loop with a single
/// scalar tail iteration
bool isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
- const ElementCount &ExactTC, unsigned int UserIC);
+ const ElementCount &ExactTC,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c1644a4d43359..f509c00f352a2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3021,7 +3021,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
- auto GetLoopRemainder = [this](unsigned MaxVF, unsigned EffectiveIC)-> const SCEV * {
+ auto GetLoopRemainder = [this](unsigned MaxVF,
+ unsigned EffectiveIC) -> const SCEV * {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
@@ -3047,7 +3048,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (MaxPowerOf2RuntimeVF > 0u) {
assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
"MaxFixedVF must be a power of 2");
- if (const auto *Rem = GetLoopRemainder(*MaxPowerOf2RuntimeVF, EffectiveIC); Rem && Rem->isZero()) {
+ if (const auto *Rem = GetLoopRemainder(*MaxPowerOf2RuntimeVF, EffectiveIC);
+ Rem && Rem->isZero()) {
// Accept MaxFixedVF if we do not have a tail.
LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
return MaxFactors;
@@ -3058,11 +3060,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
- if (MaxPowerOf2RuntimeVF > 0u && EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop) {
+ if (MaxPowerOf2RuntimeVF > 0u &&
+ EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop) {
// If we have a low-trip-count, and the fixed-width VF is known to divide
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
- if (const auto *Rem = GetLoopRemainder(MaxFactors.FixedVF.getFixedValue(), EffectiveIC); Rem && Rem->isZero()) {
+ if (const auto *Rem =
+ GetLoopRemainder(MaxFactors.FixedVF.getFixedValue(), EffectiveIC);
+ Rem && Rem->isZero()) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
@@ -3077,10 +3082,12 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If a function is marked as minsize/optsize or OptForSize is set, do not
// allow this form of transformation as this will increase CodeSize.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
- if (ExactTC.getFixedValue() > 1 && !TheFunction->hasOptSize() && !Config.OptForSize) {
+ if (ExactTC.getFixedValue() > 1 && !TheFunction->hasOptSize() &&
+ !Config.OptForSize) {
unsigned TC = ExactTC.getFixedValue();
unsigned VF = (TC - 1) / EffectiveIC;
- if (const auto *Rem = GetLoopRemainder(VF, EffectiveIC); Rem && Rem->isOne() && isPowerOf2_32(VF)) {
+ if (const auto *Rem = GetLoopRemainder(VF, EffectiveIC);
+ Rem && Rem->isOne() && isPowerOf2_32(VF)) {
LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
<< " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(VF);
@@ -5826,8 +5833,9 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
- const ElementCount &ExactTC, unsigned int UserIC) {
+bool LoopVectorizationPlanner::isProfitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, const ElementCount &ExactTC,
+ unsigned int UserIC) {
// VectorCost reflects the cost of the requried vector iteration(s) and the
// one remaining scalar iteration cost
unsigned TC = ExactTC.getFixedValue();
@@ -5854,7 +5862,8 @@ bool LoopVectorizationPlanner::shouldConsiderOneScalarTail(
unsigned int UserIC) {
if (ExactTC.isFixed()) {
unsigned TC = ExactTC.getFixedValue();
- if (!CurrentFactor.Width.isScalable() && TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold())
+ if (!CurrentFactor.Width.isScalable() && TC != 0 &&
+ TC <= TTI.getMinTripCountTailFoldingThreshold())
return true;
}
return false;
More information about the llvm-commits
mailing list