[llvm] [LoopVectorize] Improve Vectorization of Low Trip Count Loops (PR #195823)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 08:45:59 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/195823
>From 58e371800dc1dd67a177118e6c1c0c6378260c3e Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 1 May 2026 11:43:47 +0100
Subject: [PATCH 01/17] [LoopVectorize] Improve Vectorization of Small Loops
Currently, Small Loops with Trip Counts less than 16, and in
situations where the Trip Count (TC) is less than the Tail Folding
Threshold are harder to vectorize, its only possible where no epilogue
will be emitted. However, for loops with large bodies and small trip
counts this can be counterprodictive to performance, often failing to
vectorize entirely. This is more prevelant with targets where
`getMinTripCountTailFoldingThreshold()` returns a value greater than 0.
To address this, the Small Loops where the TC == VF + 1 can now vectorize,
leading to a single vectorized itneration and a single scalar iteration.
Later passes can then remove the loop's entirely.
Testing an with OpenSource Fortran HPC Benchmark which includes multiple
loops with small trip counts, but large loop bodies, has shown significant
improvement to runtime after these changes.
Assisted-by: Claude Sonnet 4.6/Codex
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++-
.../AArch64/sve-low-trip-count.ll | 24 +--
.../sve-small-trip-count-vf-plus-one.ll | 195 ++++++++++++++++++
3 files changed, 243 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c03a9b20bcd19..df85d13c49dab 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3064,6 +3064,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
+ auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
+ unsigned int FixedVF, unsigned int ScalableVF) {
+ MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
+ };
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3075,9 +3084,36 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- MaxFactors.ScalableVF = ElementCount::getScalable(0);
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
return MaxFactors;
}
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // 2 between 2 and MaxVF.
+ //
+ // This produces 1 vector iteration, and 1 scalar iteration with
+ // no remainder. Later passes will eliminate the loop and leave
+ // straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
+ ExactTC && ExactTC.isFixed()) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
+ "iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ return MaxFactors;
+ }
+ // If the maximum VF cannot produce 1 vector iteration + 1 scalar
+ // iteration, step down VF's to find one that can.
+ for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
+ VF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ << " with 1 scalar iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, VF, 0);
+ return MaxFactors;
+ }
+ }
+ }
}
reportVectorizationFailure(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index c36daa40f6193..76642c63dbd26 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -56,22 +56,20 @@ exit:
define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
; CHECK-LABEL: define void @trip5_i8(
; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
-; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
-; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..71157580334d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
+; produces one vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
+; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
+define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_vf2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=4: exact multiple of VF=4. Vectorizes via the original
+; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
+define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc4_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 4
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From e453f50232dd652e25f4a3dc598cc7c1b144cae2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 11:29:55 +0100
Subject: [PATCH 02/17] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index df85d13c49dab..80d9551b69b34 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3070,7 +3070,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
};
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
+ unsigned int VF) -> bool {
return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3096,7 +3097,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ if (HasOneScalarIterationRemainder(
+ ExactTC, MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
"iteration remainder.\n");
ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
>From 7becb214927746d44e787cad7209eeed2746942d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 15:50:14 +0100
Subject: [PATCH 03/17] Responding to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++---
.../AArch64/sve-low-trip-count.ll | 38 ------
.../sve-small-trip-count-vf-plus-one.ll | 86 +-------------
.../RISCV/small-trip-count-vf-plus-one.ll | 109 ++++++++++++++++++
4 files changed, 127 insertions(+), 144 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 80d9551b69b34..c281d75424511 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3064,15 +3064,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
- unsigned int FixedVF, unsigned int ScalableVF) {
- MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
- MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
- };
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int VF) -> bool {
- return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ unsigned int MaxVF) -> bool {
+ return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
@@ -3085,7 +3080,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
// Allow cases where the ExactTC == VF + 1. VF can be any power of
@@ -3096,22 +3091,21 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(
- ExactTC, MaxFactors.FixedVF.getFixedValue())) {
- LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
- "iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
- return MaxFactors;
- }
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can.
- for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
- VF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ // iteration, step down VF's to find one that can. The result should
+ // also eliminate any loops.
+ //
+ // Forced interleaving is considered when seeing if OneScalarIterationRemainder
+ // is produced. It may prodiced more than one vector iteration, but only one
+ // scalar iteration.
+ for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
+ MaxVF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, VF, 0);
+ MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index 76642c63dbd26..03746ee82223a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -53,42 +53,4 @@ exit:
ret void
}
-define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
-; CHECK-LABEL: define void @trip5_i8(
-; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
-; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
- %0 = load i8, ptr %gep.src, align 1
- %mul = shl i8 %0, 1
- %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
- %1 = load i8, ptr %gep.dst, align 1
- %add = add i8 %mul, %1
- store i8 %add, ptr %gep.dst, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 5
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 71157580334d1..711ce3bc439d2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
-; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
-; produces one vector iteration and one scalar iteration.
+; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
;
; RUN: opt -S -p loop-vectorize %s | FileCheck %s
@@ -107,88 +107,6 @@ exit:
ret void
}
-; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
-; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
-define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc3_vf2_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
-; TC=4: exact multiple of VF=4. Vectorizes via the original
-; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
-define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc4_vf4_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 4
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..39917f15e2870
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on RISCV where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s -mtriple=riscv64 -mattr=+v -tail-folding-policy=dont-fold-tail | FileCheck %s
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From f6aa0264836fef2f203b046ab36e4111908b3698 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 16:30:20 +0100
Subject: [PATCH 04/17] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c281d75424511..9b222c5313aff 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3065,8 +3065,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int MaxVF) -> bool {
+ auto HasOneScalarIterationRemainder =
+ [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3095,10 +3095,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if OneScalarIterationRemainder
- // is produced. It may prodiced more than one vector iteration, but only one
- // scalar iteration.
+ //
+ // Forced interleaving is considered when seeing if
+ // OneScalarIterationRemainder is produced. It may prodiced more than
+ // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
>From 5f8dc7d77c07725b6b790c8aa3a06db91c2355e8 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 6 May 2026 08:56:45 +0100
Subject: [PATCH 05/17] Use ExpectedTC
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9b222c5313aff..da86fd5f29177 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3083,15 +3083,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExpectedTC->isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3101,7 +3100,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
>From 585505ec2098b12d9e5897311ca011e2a23db746 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 10:12:06 +0100
Subject: [PATCH 06/17] Update OPT Test and readd ExactTc
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../sve-small-trip-count-vf-plus-one.ll | 184 ++++++++++++++++--
2 files changed, 170 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index da86fd5f29177..9b222c5313aff 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3083,14 +3083,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExpectedTC->isFixed()) {
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3100,7 +3101,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 711ce3bc439d2..7e93ae7f518b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -13,9 +13,9 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
@@ -23,11 +23,11 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -35,7 +35,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -55,14 +55,110 @@ exit:
ret void
}
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; the low-trip-count path steps down to a smaller profitable VF.
+define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[A]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i16> [[WIDE_LOAD]], splat (i16 1)
+; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[B]], align 2
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-NEXT: [[ADD:%.*]] = add i16 [[VAL]], 1
+; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_B]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i16, ptr %b, i64 %iv
+ %val = load i16, ptr %gep.a, align 2
+ %add = add i16 %val, 1
+ store i16 %add, ptr %gep.b, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; search can step down more than once before accepting VF=4.
+define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[VAL]], 1
+; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %val = load i8, ptr %gep.a, align 1
+ %add = add i8 %val, 1
+ store i8 %add, ptr %gep.b, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
@@ -75,11 +171,11 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -87,7 +183,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -107,6 +203,58 @@ exit:
ret void
}
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i64(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[WIDE_LOAD1]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[VAL]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %val = load i64, ptr %gep.a, align 4
+ %add = add nsw i64 %val, 1
+ store i64 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 9ff11cbf1fe0e6c392f88d739bb5ad11f0ca1242 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 14:42:25 +0100
Subject: [PATCH 07/17] Check for ExactTC!=0
The checks for matching to VF+1 == TC should protect against this, but
its best to the explicit.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 53 +++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9b222c5313aff..0a04dde3393ef 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3091,7 +3091,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExactTC.getFixedValue() != 0) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 7e93ae7f518b3..4b80ffb894817 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -255,6 +255,59 @@ exit:
ret void
}
+; ExactTC is unknown here because the loop trip count is the runtime value %n,
+; but the guard proves the maximum trip count is 5. This should still take the
+; low-trip-count path, but it must not use the VF+1 escape because
+; getSmallConstantTripCount returns 0.
+define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
+; CHECK-LABEL: define void @unknown_exact_tc_max5(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IS_ZERO:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[IS_ZERO]], label %[[EXIT:.*]], label %[[GUARD:.*]]
+; CHECK: [[GUARD]]:
+; CHECK-NEXT: [[TOO_LARGE:%.*]] = icmp ugt i64 [[N]], 5
+; CHECK-NEXT: br i1 [[TOO_LARGE]], label %[[EXIT]], label %[[LOOP_PREHEADER:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %is.zero = icmp eq i64 %n, 0
+ br i1 %is.zero, label %exit, label %guard
+
+guard:
+ %too.large = icmp ugt i64 %n, 5
+ br i1 %too.large, label %exit, label %loop
+
+loop:
+ %iv = phi i64 [ 0, %guard ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 82ee2d4b19b58b9801861313e6d0c0424408fc86 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 19 May 2026 11:23:52 +0100
Subject: [PATCH 08/17] Respond to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 88 +++++++++++++++++--
2 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0a04dde3393ef..209cbbff23348 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3097,7 +3097,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// also eliminate any loops.
//
// Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may prodiced more than
+ // OneScalarIterationRemainder is produced. It may produced more than
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 4b80ffb894817..d9db7d31d3b87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,17 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
; produces the required number of vector iteration and one scalar iteration.
-;
-; RUN: opt -S -p loop-vectorize %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
-define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_vf4_vectorize(
+define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -41,6 +40,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -51,6 +51,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -89,6 +90,7 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
@@ -99,6 +101,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -137,6 +140,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
@@ -147,14 +151,15 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
-define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -189,6 +194,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -199,6 +205,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
@@ -241,6 +248,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -251,14 +259,14 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. This should still take the
-; low-trip-count path, but it must not use the VF+1 escape because
-; getSmallConstantTripCount returns 0.
+; but the guard proves the maximum trip count is 5. it must not use the VF+1
+; escape because getSmallConstantTripCount returns 0.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -308,7 +316,69 @@ exit:
ret void
}
+; TC=5, VF=4, UserIC=4
+; The user interleave count should be ignored because the dependence distance
+; makes the loop unsafe for interleaving > 1. Vectorization should still pick
+; VF=4 and produce 1 vector iteration plus 1 scalar iteration.
+define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_unsafe_useric_distance4_i32(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP0]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B_DST:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[B_SRC:%.*]] = getelementptr inbounds i32, ptr [[B_DST]], i64 -4
+; CHECK-NEXT: [[DEP:%.*]] = load i32, ptr [[B_SRC]], align 4
+; CHECK-NEXT: [[A_SRC:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[A_SRC]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], [[DEP]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %b.dst = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b.src = getelementptr inbounds i32, ptr %b.dst, i64 -4
+ %dep = load i32, ptr %b.src, align 4
+ %a.src = getelementptr inbounds i32, ptr %a, i64 %iv
+ %val = load i32, ptr %a.src, align 4
+ %add = add nsw i32 %val, %dep
+ store i32 %add, ptr %b.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 9
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = distinct !{!2, !3, !4}
+!3 = !{!"llvm.loop.interleave.count", i32 4}
+!4 = !{!"llvm.loop.vectorize.width", i32 4}
>From 0c88e0991c851ec01956376d283d5f22a338105d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:23:41 +0000
Subject: [PATCH 09/17] Reuse EffectiveIC in NoScalarEpilogueNeeded
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 209cbbff23348..8308468abcd3b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3030,13 +3030,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
- auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
+ auto NoScalarEpilogueNeeded = [this](unsigned MaxVF, unsigned EffectiveIC) {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
!Legal->hasUncountableEarlyExit())
return false;
- unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
+ unsigned MaxVFtimesIC = MaxVF * EffectiveIC;
ScalarEvolution *SE = PSE.getSE();
// Calling getSymbolicMaxBackedgeTakenCount enables support for loops
// with uncountable exits. For countable loops, the symbolic maximum must
@@ -3053,10 +3053,11 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return Rem->isZero();
};
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
if (MaxPowerOf2RuntimeVF > 0u) {
assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
"MaxFixedVF must be a power of 2");
- if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
+ if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF, EffectiveIC)) {
// Accept MaxFixedVF if we do not have a tail.
LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
return MaxFactors;
@@ -3064,7 +3065,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder =
[EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
@@ -3077,7 +3077,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 3ea49ef441def7bd2d3bb8625c66c98d82211311 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:51:36 +0000
Subject: [PATCH 10/17] format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8308468abcd3b..972041825bd5e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3077,7 +3077,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(),
+ EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 95e204bf13735dd873ebf2326bc3f73d7e67aa82 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 28 May 2026 11:51:40 +0100
Subject: [PATCH 11/17] Address nit comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++-----
.../AArch64/sve-small-trip-count-vf-plus-one.ll | 10 +++++-----
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 972041825bd5e..7bb0619636e6d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3096,15 +3096,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may produced more than
- // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
+ // OneScalarIterationRemainder takes account of any forced
+ // interleaving.
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
- << " with 1 scalar iteration remainder.\n");
+ << " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index d9db7d31d3b87..5c08f97a55aa2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -57,7 +57,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; VF=8 is a natural fixed-width for i16 types on AArch64, so this also checks that
; the low-trip-count path steps down to a smaller profitable VF.
define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
@@ -107,7 +107,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; VF=16 is a natural fixed-width for i8 types on AArch64, so this checks that the
; search can step down more than once before accepting VF=4.
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
@@ -264,9 +264,9 @@ exit:
ret void
}
-; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. it must not use the VF+1
-; escape because getSmallConstantTripCount returns 0.
+; In this case the vectoriser shouldn't optimise for a single vector iteration
+; + single scalar iteration, because there is no guarantee we will enter the vector
+; loop.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
>From 8e08177144a2647180c48c9b584e071c78885b79 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 09:54:07 +0100
Subject: [PATCH 12/17] Add cost modelling for where scalar loops are more
profitable
---
.../Vectorize/LoopVectorizationPlanner.cpp | 42 +++---
.../Vectorize/LoopVectorizationPlanner.h | 5 +
.../Transforms/Vectorize/LoopVectorize.cpp | 77 ++++++++++
.../sve-small-trip-count-vf-plus-one-cost.ll | 132 ++++++++++++++++++
4 files changed, 237 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index dbb5ad28fb4ed..f95a5c1f1a381 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -742,25 +742,8 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
if (!MaxTripCount)
return LowerCostWithoutTC;
- auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
- InstructionCost VectorCost,
- InstructionCost ScalarCost) {
- // If the trip count is a known (possibly small) constant, the trip count
- // will be rounded up to an integer number of iterations under
- // FoldTailByMasking. The total cost in that case will be
- // VecCost*ceil(TripCount/VF). When not folding the tail, the total
- // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
- // some extra overheads, but for the purpose of comparing the costs of
- // different VFs we can use this to compare the total loop-body cost
- // expected after vectorization.
- if (HasTail)
- return VectorCost * (MaxTripCount / VF) +
- ScalarCost * (MaxTripCount % VF);
- return VectorCost * divideCeil(MaxTripCount, VF);
- };
-
- auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
- auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
+ auto RTCostA = getCostForKnownTripCount(A, MaxTripCount, HasTail);
+ auto RTCostB = getCostForKnownTripCount(B, MaxTripCount, HasTail);
bool LowerCostWithTC = CmpFn(RTCostA, RTCostB);
LLVM_DEBUG(if (LowerCostWithTC != LowerCostWithoutTC) {
dbgs() << "LV: VF " << (LowerCostWithTC ? A.Width : B.Width)
@@ -782,6 +765,27 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
IsEpilogue);
}
+InstructionCost LoopVectorizationPlanner::getCostForKnownTripCount(
+ const VectorizationFactor &VF, unsigned TripCount, bool HasTail) const {
+ unsigned EstimatedWidth = VF.Width.getKnownMinValue();
+ if (std::optional<unsigned> VScale = Config.getVScaleForTuning())
+ if (VF.Width.isScalable())
+ EstimatedWidth *= *VScale;
+
+ // If the trip count is a known (possibly small) constant, the trip count
+ // will be rounded up to an integer number of iterations under
+ // FoldTailByMasking. The total cost in that case will be
+ // VecCost*ceil(TripCount/VF). When not folding the tail, the total
+ // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
+ // some extra overheads, but for the purpose of comparing the costs of
+ // different VFs we can use this to compare the total loop-body cost
+ // expected after vectorization.
+ if (HasTail)
+ return VF.Cost * (TripCount / EstimatedWidth) +
+ VF.ScalarCost * (TripCount % EstimatedWidth);
+ return VF.Cost * divideCeil(TripCount, EstimatedWidth);
+}
+
// TODO: we could return a pair of values that specify the max VF and
// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b39ffdebf6179..47b7fb86c2e26 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -1004,6 +1004,11 @@ class LoopVectorizationPlanner {
const unsigned MaxTripCount, bool HasTail,
bool IsEpilogue = false) const;
+ /// Returns the estimated loop-body cost for \p VF and a known trip count.
+ InstructionCost getCostForKnownTripCount(const VectorizationFactor &VF,
+ unsigned TripCount,
+ bool HasTail) const;
+
/// Determines if we have the infrastructure to vectorize the loop and its
/// epilogue, assuming the main loop is vectorized by \p MainPlan.
bool isCandidateForEpilogueVectorization(VPlan &MainPlan) const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7bb0619636e6d..ee55883a32f4c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5829,6 +5829,46 @@ LoopVectorizationPlanner::computeBestVF() {
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
+ bool ForceVectorization =
+ Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
+ hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned EstimatedWidth =
+ estimateElementCount(UserVF, Config.getVScaleForTuning());
+ if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
+ TC == EstimatedWidth + 1) {
+ ElementCount ScalarVF = ElementCount::getFixed(1);
+ InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ LLVM_DEBUG(dbgs()
+ << "LV: Scalar loop costs: " << ScalarCost << ".\n");
+
+ InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
+ VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
+ VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces
+ // extra control flow and a scalar epilogue for a single element, so
+ // require the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
+ AdjustedVectorCost >= ScalarCostForTC) {
+ LLVM_DEBUG(dbgs()
+ << "LV: Rejecting VF " << UserVF
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return {ScalarFactor, &FirstPlan};
+ }
+ }
+ }
+ }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -5871,6 +5911,40 @@ LoopVectorizationPlanner::computeBestVF() {
}
VPlan *PlanForBestVF = &FirstPlan;
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC % EstimatedWidth != 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -5907,6 +5981,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ continue;
+
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
BestFactor = CurrentFactor;
PlanForBestVF = P.get();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
new file mode 100644
index 0000000000000..82cbb431ed76f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -0,0 +1,132 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -S %s | FileCheck %s --check-prefix=IR
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s --check-prefix=DBG
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body:
+;
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
+; DBG: LV: Using user VF 2.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 10.
+; DBG: Cost for VF 2: 15
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
+; DBG-NOT: Rejecting VF 2
+; DBG: LV: Selecting VF: 2.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+declare i8 @llvm.smin.i8(i8, i8)
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.width", i32 2}
>From 73cb9bb838810711e5421b98457121199de9844d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 14:06:50 +0100
Subject: [PATCH 13/17] Remove for loop for calculating best VF
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 +++++++------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ee55883a32f4c..d989873c50c7b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3065,10 +3065,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto HasOneScalarIterationRemainder =
- [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
- return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
- };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3092,18 +3088,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.getFixedValue() != 0) {
- // If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can. The result should
- // also eliminate any loops.
- for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
- MaxVF /= 2) {
- // OneScalarIterationRemainder takes account of any forced
- // interleaving.
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
+ ExactTC.getFixedValue() > 1) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned MaxFixedVF = MaxFactors.FixedVF.getFixedValue();
+ if ((TC - 1) % EffectiveIC == 0) {
+ unsigned VF = (TC - 1) / EffectiveIC;
+ if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
<< " with 1 scalar iteration remaining.\n");
- MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
@@ -5924,7 +5917,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC % EstimatedWidth != 1)
+ if (TC != EstimatedWidth + 1)
return false;
InstructionCost VectorCost =
>From e6129ebc268253494ba1709bf9790d227d4455da Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 23 Jun 2026 09:30:56 +0100
Subject: [PATCH 14/17] Refactor IsUnprofitableOneScalarTail Lambda Function
---
.../Transforms/Vectorize/LoopVectorize.cpp | 92 +++++++++----------
1 file changed, 42 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d989873c50c7b..c9af15e7e08b4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5814,6 +5814,42 @@ LoopVectorizationPlanner::computeBestVF() {
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ const InstructionCost &ScalarCost) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != EstimatedWidth + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!AdjustedVectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
ElementCount UserVF = Hints.getWidth();
if (VPlans.size() == 1) {
@@ -5840,23 +5876,10 @@ LoopVectorizationPlanner::computeBestVF() {
InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces
- // extra control flow and a scalar epilogue for a single element, so
- // require the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
- AdjustedVectorCost >= ScalarCostForTC) {
- LLVM_DEBUG(dbgs()
- << "LV: Rejecting VF " << UserVF
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
+ if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
+ ForceVectorization, ExactTC,
+ ScalarFactor, ScalarCost)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -5905,39 +5928,6 @@ LoopVectorizationPlanner::computeBestVF() {
VPlan *PlanForBestVF = &FirstPlan;
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -5974,7 +5964,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor,
+ ScalarCost))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
>From 54445e18410f15bee2bb05047f6e36e961fc09b8 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 2 Jul 2026 15:54:45 +0100
Subject: [PATCH 15/17] Add consideration of IC and test
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 ++--
.../sve-small-trip-count-vf-plus-one-cost.ll | 46 ++++++++++++++++---
2 files changed, 44 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c9af15e7e08b4..7c63c94e70df7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5818,7 +5818,7 @@ LoopVectorizationPlanner::computeBestVF() {
[&](const VectorizationFactor &CurrentFactor, bool HasTail,
bool ForceVectorization, const ElementCount &ExactTC,
const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost) {
+ const InstructionCost &ScalarCost, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5829,7 +5829,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
+ if (TC != (EstimatedWidth * UserIC) + 1)
return false;
InstructionCost VectorCost =
@@ -5852,6 +5852,7 @@ LoopVectorizationPlanner::computeBestVF() {
};
ElementCount UserVF = Hints.getWidth();
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
@@ -5879,7 +5880,7 @@ LoopVectorizationPlanner::computeBestVF() {
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost)) {
+ ScalarFactor, ScalarCost, UserIC)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -5966,7 +5967,7 @@ LoopVectorizationPlanner::computeBestVF() {
if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost))
+ ScalarCost, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 82cbb431ed76f..70eb5b07c25aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -7,13 +7,7 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc3_udiv_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_forced(
-; IR: vector.body:
+; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
@@ -44,6 +38,9 @@ exit:
define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
; DBG: LV: Using user VF 2.
; DBG: LV: Scalar loop costs: 9.
@@ -71,6 +68,9 @@ exit:
}
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
@@ -99,6 +99,9 @@ exit:
define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
; DBG-NOT: Rejecting VF 2
; DBG: LV: Selecting VF: 2.
@@ -122,6 +125,33 @@ exit:
ret void
}
+define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0{
+; IR-LABEL: define void @tc5_udiv_i8_reject_ic2(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
+; DBG-NOT: LV: Selecting VF: 2.
+; DBG Rejecting VF 2
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+
+exit:
+ ret void
+}
+
declare i8 @llvm.smin.i8(i8, i8)
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
@@ -130,3 +160,5 @@ attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.vectorize.width", i32 2}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.interleave.count", i32 2}
>From aaf7100ce99909116a5f9c12b0b3a764863781a2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:25:20 +0100
Subject: [PATCH 16/17] Respond to review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 7 ++
.../Transforms/Vectorize/LoopVectorize.cpp | 109 +++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 75 ++++++------
.../sve-small-trip-count-vf-plus-one.ll | 81 +++++++++++++
4 files changed, 166 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 47b7fb86c2e26..6801dfcd9c64f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -878,6 +878,13 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
+ /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
+ /// it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7c63c94e70df7..786c238009290 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3094,7 +3094,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if ((TC - 1) % EffectiveIC == 0) {
unsigned VF = (TC - 1) / EffectiveIC;
if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
<< " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
@@ -5808,84 +5808,57 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != (EstimatedWidth * UserIC) + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
+ if (!VectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+}
+
std::pair<VectorizationFactor, VPlan *>
LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost, unsigned int UserIC) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != (EstimatedWidth * UserIC) + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!AdjustedVectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
ElementCount UserVF = Hints.getWidth();
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
- bool ForceVectorization =
- Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
- hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
- unsigned TC = ExactTC.getFixedValue();
- unsigned EstimatedWidth =
- estimateElementCount(UserVF, Config.getVScaleForTuning());
- if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
- TC == EstimatedWidth + 1) {
- ElementCount ScalarVF = ElementCount::getFixed(1);
- InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
- LLVM_DEBUG(dbgs()
- << "LV: Scalar loop costs: " << ScalarCost << ".\n");
-
- InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
- VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
- VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
- ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost, UserIC)) {
- return {ScalarFactor, &FirstPlan};
- }
- }
- }
- }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -5965,9 +5938,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost, UserIC))
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 70eb5b07c25aa..4516062f9b7d1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -10,7 +10,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
@@ -36,43 +36,12 @@ exit:
ret void
}
-define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
- ptr noalias %c) #0 {
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-
-; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
-; DBG: LV: Using user VF 2.
-; DBG: LV: Scalar loop costs: 9.
-; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
-; DBG: LV: Vectorization is possible but not beneficial.
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %pa = getelementptr inbounds i8, ptr %a, i64 %iv
- %pb = getelementptr inbounds i8, ptr %b, i64 %iv
- %pc = getelementptr inbounds i8, ptr %c, i64 %iv
- %va = load i8, ptr %pa, align 1
- %vb = load i8, ptr %pb, align 1
- %div = udiv i8 %va, %vb
- store i8 %div, ptr %pc, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
-
-exit:
- ret void
-}
-
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
; IR-NOT: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
@@ -131,7 +100,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG Rejecting VF 2
+; DBG: Rejecting VF 2
entry:
br label %loop
@@ -146,7 +115,37 @@ loop:
store i8 %div, ptr %pc, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc5_sin_f32_select_smaller_vf(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
+; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
+; DBG: LV: Rejecting VF 4 for one-scalar-tail low trip count: vector cost 88 >= scalar cost 80.
+; DBG-NOT: Selecting VF: 4
+; DBG: LV: Selecting VF: 2.
+
+
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds float, ptr %a, i64 %iv
+ %pc = getelementptr inbounds float, ptr %c, i64 %iv
+ %va = load float, ptr %pa, align 4
+ %sin = tail call float @llvm.sin.f32(float %va)
+ store float %sin, ptr %pc, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
exit:
ret void
@@ -154,11 +153,11 @@ exit:
declare i8 @llvm.smin.i8(i8, i8)
+declare float @llvm.sin.f32(float)
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
-!3 = !{!"llvm.loop.vectorize.width", i32 2}
-!4 = distinct !{!4, !5}
-!5 = !{!"llvm.loop.interleave.count", i32 2}
+!3 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 5c08f97a55aa2..b999acc21309c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -375,6 +375,87 @@ exit:
ret void
}
+; For this example, vectorization should not take place as the
+; scalar cost is better than the vectorized cost.
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+; CHECK-LABEL: define void @tc3_udiv_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[PB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[PC:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT: [[VA:%.*]] = load i8, ptr [[PA]], align 1
+; CHECK-NEXT: [[VB:%.*]] = load i8, ptr [[PB]], align 1
+; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[VA]], [[VB]]
+; CHECK-NEXT: store i8 [[DIV]], ptr [[PC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+ ptr noalias %c) #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_smin_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[TMP1]], i8 [[TMP2]])
+; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From ecf68082ad3a46259c3dada8e0a5b03d312a74f2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:55:34 +0100
Subject: [PATCH 17/17] format
---
.../Vectorize/LoopVectorizationPlanner.h | 14 +++++----
.../Transforms/Vectorize/LoopVectorize.cpp | 29 +++++++++++--------
2 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 6801dfcd9c64f..127664101a839 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -878,12 +878,14 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
- /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
- /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
- /// it is better to leave the original Scalar loop in place.
- bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that
+ /// leaves a Vector iteration, followed by a single iteration scalar tail. For
+ /// some uses cases, it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ bool HasTail, bool ForceVectorization,
+ const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 786c238009290..a269c01a43091 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5808,9 +5808,10 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5819,8 +5820,8 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
return false;
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
+ unsigned EstimatedWidth =
+ estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
if (TC != (EstimatedWidth * UserIC) + 1)
return false;
@@ -5832,11 +5833,13 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
// control flow and a scalar epilogue for a single element, so require
// the vectorized form to save at least one scalar iteration.
InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
- if (!VectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC) {
- LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
- return false;
- }
+ if (!VectorCost.isValid() || AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " < " << ScalarCostForTC
+ << ".\n");
+ return false;
+ }
LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
@@ -5938,9 +5941,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ unsigned int UserIC =
+ Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor, UserIC))
+ ForceVectorization, ExactTC, ScalarFactor,
+ UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
More information about the llvm-commits
mailing list