[llvm] [LoopVectorize] Improve Vectorization of Low Trip Count Loops (PR #195823)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 02:26:11 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/195823
>From 58e371800dc1dd67a177118e6c1c0c6378260c3e Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 1 May 2026 11:43:47 +0100
Subject: [PATCH 01/22] [LoopVectorize] Improve Vectorization of Small Loops
Currently, Small Loops with Trip Counts less than 16, and in
situations where the Trip Count (TC) is less than the Tail Folding
Threshold are harder to vectorize, its only possible where no epilogue
will be emitted. However, for loops with large bodies and small trip
counts this can be counterprodictive to performance, often failing to
vectorize entirely. This is more prevelant with targets where
`getMinTripCountTailFoldingThreshold()` returns a value greater than 0.
To address this, the Small Loops where the TC == VF + 1 can now vectorize,
leading to a single vectorized itneration and a single scalar iteration.
Later passes can then remove the loop's entirely.
Testing an with OpenSource Fortran HPC Benchmark which includes multiple
loops with small trip counts, but large loop bodies, has shown significant
improvement to runtime after these changes.
Assisted-by: Claude Sonnet 4.6/Codex
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++-
.../AArch64/sve-low-trip-count.ll | 24 +--
.../sve-small-trip-count-vf-plus-one.ll | 195 ++++++++++++++++++
3 files changed, 243 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c03a9b20bcd19..df85d13c49dab 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3064,6 +3064,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
+ auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
+ unsigned int FixedVF, unsigned int ScalableVF) {
+ MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
+ };
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3075,9 +3084,36 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- MaxFactors.ScalableVF = ElementCount::getScalable(0);
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
return MaxFactors;
}
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // 2 between 2 and MaxVF.
+ //
+ // This produces 1 vector iteration, and 1 scalar iteration with
+ // no remainder. Later passes will eliminate the loop and leave
+ // straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
+ ExactTC && ExactTC.isFixed()) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
+ "iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ return MaxFactors;
+ }
+ // If the maximum VF cannot produce 1 vector iteration + 1 scalar
+ // iteration, step down VF's to find one that can.
+ for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
+ VF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ << " with 1 scalar iteration remainder.\n");
+ ApplyVectorWidth(MaxFactors, VF, 0);
+ return MaxFactors;
+ }
+ }
+ }
}
reportVectorizationFailure(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index c36daa40f6193..76642c63dbd26 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -56,22 +56,20 @@ exit:
define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
; CHECK-LABEL: define void @trip5_i8(
; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
-; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
-; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..71157580334d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
+; produces one vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
+; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
+define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_vf2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=4: exact multiple of VF=4. Vectorizes via the original
+; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
+define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc4_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 4
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From e453f50232dd652e25f4a3dc598cc7c1b144cae2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 11:29:55 +0100
Subject: [PATCH 02/22] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index df85d13c49dab..80d9551b69b34 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3070,7 +3070,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
};
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC, unsigned int VF)-> bool {
+ auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
+ unsigned int VF) -> bool {
return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3096,7 +3097,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxFactors.FixedVF.getFixedValue())) {
+ if (HasOneScalarIterationRemainder(
+ ExactTC, MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
"iteration remainder.\n");
ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
>From 7becb214927746d44e787cad7209eeed2746942d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 15:50:14 +0100
Subject: [PATCH 03/22] Responding to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++---
.../AArch64/sve-low-trip-count.ll | 38 ------
.../sve-small-trip-count-vf-plus-one.ll | 86 +-------------
.../RISCV/small-trip-count-vf-plus-one.ll | 109 ++++++++++++++++++
4 files changed, 127 insertions(+), 144 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 80d9551b69b34..c281d75424511 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3064,15 +3064,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto ApplyVectorWidth = [](FixedScalableVFPair &MaxFactors,
- unsigned int FixedVF, unsigned int ScalableVF) {
- MaxFactors.FixedVF = ElementCount::getFixed(FixedVF);
- MaxFactors.ScalableVF = ElementCount::getScalable(ScalableVF);
- };
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int VF) -> bool {
- return ExactTC.getFixedValue() == ((VF * EffectiveIC) + 1);
+ unsigned int MaxVF) -> bool {
+ return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
@@ -3085,7 +3080,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
// Allow cases where the ExactTC == VF + 1. VF can be any power of
@@ -3096,22 +3091,21 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC && ExactTC.isFixed()) {
- if (HasOneScalarIterationRemainder(
- ExactTC, MaxFactors.FixedVF.getFixedValue())) {
- LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width with 1 scalar "
- "iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, MaxFactors.FixedVF.getFixedValue(), 0);
- return MaxFactors;
- }
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can.
- for (unsigned VF = MaxFactors.FixedVF.getFixedValue(); VF >= 2;
- VF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ // iteration, step down VF's to find one that can. The result should
+ // also eliminate any loops.
+ //
+ // Forced interleaving is considered when seeing if OneScalarIterationRemainder
+ // is produced. It may prodiced more than one vector iteration, but only one
+ // scalar iteration.
+ for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
+ MaxVF /= 2) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
- ApplyVectorWidth(MaxFactors, VF, 0);
+ MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index 76642c63dbd26..03746ee82223a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -53,42 +53,4 @@ exit:
ret void
}
-define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
-; CHECK-LABEL: define void @trip5_i8(
-; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = shl <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[DST]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[TMP0]], [[WIDE_LOAD1]]
-; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[DST]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
- %0 = load i8, ptr %gep.src, align 1
- %mul = shl i8 %0, 1
- %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
- %1 = load i8, ptr %gep.dst, align 1
- %add = add i8 %mul, %1
- store i8 %add, ptr %gep.dst, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 5
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 71157580334d1..711ce3bc439d2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
-; on AArch64+SVE where getMinTripCountTailFoldingThreshold() returns 5. This
-; produces one vector iteration and one scalar iteration.
+; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
;
; RUN: opt -S -p loop-vectorize %s | FileCheck %s
@@ -107,88 +107,6 @@ exit:
ret void
}
-; TC=3, VF=2: TC == FixedVF + 1 (3 == 2 + 1).
-; Should vectorize: 1 vector iteration of width 2, then 1 scalar iteration.
-define void @tc3_vf2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc3_vf2_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
-; TC=4: exact multiple of VF=4. Vectorizes via the original
-; "no scalar epilogue needed" path -- NOT the new TC==VF+1 path.
-define void @tc4_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc4_vf4_vectorize(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %val = load i32, ptr %gep.a, align 4
- %add = add nsw i32 %val, 1
- store i32 %add, ptr %gep.b, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 4
- br i1 %exitcond, label %exit, label %loop
-exit:
- ret void
-}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
new file mode 100644
index 0000000000000..39917f15e2870
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/small-trip-count-vf-plus-one.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+;
+; Test that a loop with trip count == VF + 1 is allowed to vectorize
+; on RISCV where under getMinTripCountTailFoldingThreshold(). This
+; produces the required number of vector iteration and one scalar iteration.
+;
+; RUN: opt -S -p loop-vectorize %s -mtriple=riscv64 -mattr=+v -tail-folding-policy=dont-fold-tail | FileCheck %s
+
+; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
+; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
+define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[WIDE_LOAD1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From f6aa0264836fef2f203b046ab36e4111908b3698 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 5 May 2026 16:30:20 +0100
Subject: [PATCH 04/22] formatting
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c281d75424511..9b222c5313aff 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3065,8 +3065,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
- auto HasOneScalarIterationRemainder = [EffectiveIC](ElementCount &ExactTC,
- unsigned int MaxVF) -> bool {
+ auto HasOneScalarIterationRemainder =
+ [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
};
if (ExpectedTC && ExpectedTC->isFixed() &&
@@ -3095,10 +3095,10 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if OneScalarIterationRemainder
- // is produced. It may prodiced more than one vector iteration, but only one
- // scalar iteration.
+ //
+ // Forced interleaving is considered when seeing if
+ // OneScalarIterationRemainder is produced. It may prodiced more than
+ // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
>From 5f8dc7d77c07725b6b790c8aa3a06db91c2355e8 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 6 May 2026 08:56:45 +0100
Subject: [PATCH 05/22] Use ExpectedTC
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9b222c5313aff..da86fd5f29177 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3083,15 +3083,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExactTC == VF + 1. VF can be any power of
+ // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExpectedTC->isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3101,7 +3100,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
>From 585505ec2098b12d9e5897311ca011e2a23db746 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 10:12:06 +0100
Subject: [PATCH 06/22] Update OPT Test and readd ExactTc
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../sve-small-trip-count-vf-plus-one.ll | 184 ++++++++++++++++--
2 files changed, 170 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index da86fd5f29177..9b222c5313aff 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3083,14 +3083,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
- // Allow cases where the ExpectedTC == VF + 1. VF can be any power of
+ // Allow cases where the ExactTC == VF + 1. VF can be any power of
// 2 between 2 and MaxVF.
//
// This produces 1 vector iteration, and 1 scalar iteration with
// no remainder. Later passes will eliminate the loop and leave
// straight-line code as the both iteration counts are statically known.
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExpectedTC->isFixed()) {
+ ExactTC.isFixed()) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
@@ -3100,7 +3101,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
- if (HasOneScalarIterationRemainder(*ExpectedTC, MaxVF)) {
+ if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
<< " with 1 scalar iteration remainder.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 711ce3bc439d2..7e93ae7f518b3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -13,9 +13,9 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
@@ -23,11 +23,11 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -35,7 +35,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -55,14 +55,110 @@ exit:
ret void
}
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; the low-trip-count path steps down to a smaller profitable VF.
+define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[A]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i16> [[WIDE_LOAD]], splat (i16 1)
+; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[B]], align 2
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-NEXT: [[ADD:%.*]] = add i16 [[VAL]], 1
+; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_B]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i16, ptr %b, i64 %iv
+ %val = load i16, ptr %gep.a, align 2
+ %add = add i16 %val, 1
+ store i16 %add, ptr %gep.b, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
+; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; search can step down more than once before accepting VF=4.
+define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[VAL]], 1
+; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %val = load i8, ptr %gep.a, align 1
+ %add = add i8 %val, 1
+ store i8 %add, ptr %gep.b, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
@@ -75,11 +171,11 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -87,7 +183,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -107,6 +203,58 @@ exit:
ret void
}
+; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
+; The forced interleave count should be considered when choosing VF.
+define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i64(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[WIDE_LOAD1]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP1]], ptr [[B]], align 4
+; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[VAL]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %val = load i64, ptr %gep.a, align 4
+ %add = add nsw i64 %val, 1
+ store i64 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 9ff11cbf1fe0e6c392f88d739bb5ad11f0ca1242 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 11 May 2026 14:42:25 +0100
Subject: [PATCH 07/22] Check for ExactTC!=0
The checks for matching to VF+1 == TC should protect against this, but
its best to the explicit.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 53 +++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9b222c5313aff..0a04dde3393ef 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3091,7 +3091,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.isFixed()) {
+ ExactTC.getFixedValue() != 0) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 7e93ae7f518b3..4b80ffb894817 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -255,6 +255,59 @@ exit:
ret void
}
+; ExactTC is unknown here because the loop trip count is the runtime value %n,
+; but the guard proves the maximum trip count is 5. This should still take the
+; low-trip-count path, but it must not use the VF+1 escape because
+; getSmallConstantTripCount returns 0.
+define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
+; CHECK-LABEL: define void @unknown_exact_tc_max5(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IS_ZERO:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[IS_ZERO]], label %[[EXIT:.*]], label %[[GUARD:.*]]
+; CHECK: [[GUARD]]:
+; CHECK-NEXT: [[TOO_LARGE:%.*]] = icmp ugt i64 [[N]], 5
+; CHECK-NEXT: br i1 [[TOO_LARGE]], label %[[EXIT]], label %[[LOOP_PREHEADER:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %is.zero = icmp eq i64 %n, 0
+ br i1 %is.zero, label %exit, label %guard
+
+guard:
+ %too.large = icmp ugt i64 %n, 5
+ br i1 %too.large, label %exit, label %loop
+
+loop:
+ %iv = phi i64 [ 0, %guard ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.a, align 4
+ %add = add nsw i32 %val, 1
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 82ee2d4b19b58b9801861313e6d0c0424408fc86 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 19 May 2026 11:23:52 +0100
Subject: [PATCH 08/22] Respond to review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../sve-small-trip-count-vf-plus-one.ll | 88 +++++++++++++++++--
2 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0a04dde3393ef..209cbbff23348 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3097,7 +3097,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// also eliminate any loops.
//
// Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may prodiced more than
+ // OneScalarIterationRemainder is produced. It may produced more than
// one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 4b80ffb894817..d9db7d31d3b87 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -1,17 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -p loop-vectorize %s | FileCheck %s
;
; Test that a loop with trip count == VF + 1 is allowed to vectorize
; on AArch64 where under getMinTripCountTailFoldingThreshold(). This
; produces the required number of vector iteration and one scalar iteration.
-;
-; RUN: opt -S -p loop-vectorize %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
; TC=5, VF=4: TC == MaxFixedVF + 1 (5 == 4 + 1).
; The new code path should trigger: 1 vectorized iteration + 1 scalar iteration.
-define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_vf4_vectorize(
+define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -41,6 +40,7 @@ define void @tc5_vf4_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -51,6 +51,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -89,6 +90,7 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i16, ptr %a, i64 %iv
@@ -99,6 +101,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
@@ -137,6 +140,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
@@ -147,14 +151,15 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop
+
exit:
ret void
}
; TC=5, VF=2, IC=2: TC == VF * IC + 1 (5 == 2 * 2 + 1).
; The forced interleave count should be considered when choosing VF.
-define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc5_forced_ic2_vectorize(
+define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*:]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
@@ -189,6 +194,7 @@ define void @tc5_forced_ic2_vectorize(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -199,6 +205,7 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
@@ -241,6 +248,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -251,14 +259,14 @@ loop:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
exit:
ret void
}
; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. This should still take the
-; low-trip-count path, but it must not use the VF+1 escape because
-; getSmallConstantTripCount returns 0.
+; but the guard proves the maximum trip count is 5. it must not use the VF+1
+; escape because getSmallConstantTripCount returns 0.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -308,7 +316,69 @@ exit:
ret void
}
+; TC=5, VF=4, UserIC=4
+; The user interleave count should be ignored because the dependence distance
+; makes the loop unsafe for interleaving > 1. Vectorization should still pick
+; VF=4 and produce 1 vector iteration plus 1 scalar iteration.
+define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_vf4_unsafe_useric_distance4_i32(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP0]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B_DST:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[B_SRC:%.*]] = getelementptr inbounds i32, ptr [[B_DST]], i64 -4
+; CHECK-NEXT: [[DEP:%.*]] = load i32, ptr [[B_SRC]], align 4
+; CHECK-NEXT: [[A_SRC:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[A_SRC]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[VAL]], [[DEP]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %b.dst = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b.src = getelementptr inbounds i32, ptr %b.dst, i64 -4
+ %dep = load i32, ptr %b.src, align 4
+ %a.src = getelementptr inbounds i32, ptr %a, i64 %iv
+ %val = load i32, ptr %a.src, align 4
+ %add = add nsw i32 %val, %dep
+ store i32 %add, ptr %b.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 9
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = distinct !{!2, !3, !4}
+!3 = !{!"llvm.loop.interleave.count", i32 4}
+!4 = !{!"llvm.loop.vectorize.width", i32 4}
>From 0c88e0991c851ec01956376d283d5f22a338105d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:23:41 +0000
Subject: [PATCH 09/22] Reuse EffectiveIC in NoScalarEpilogueNeeded
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 209cbbff23348..8308468abcd3b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3030,13 +3030,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
}
- auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
+ auto NoScalarEpilogueNeeded = [this](unsigned MaxVF, unsigned EffectiveIC) {
// Return false if the loop is neither a single-latch-exit loop nor an
// early-exit loop as tail-folding is not supported in that case.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
!Legal->hasUncountableEarlyExit())
return false;
- unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
+ unsigned MaxVFtimesIC = MaxVF * EffectiveIC;
ScalarEvolution *SE = PSE.getSE();
// Calling getSymbolicMaxBackedgeTakenCount enables support for loops
// with uncountable exits. For countable loops, the symbolic maximum must
@@ -3053,10 +3053,11 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return Rem->isZero();
};
+ unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
if (MaxPowerOf2RuntimeVF > 0u) {
assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
"MaxFixedVF must be a power of 2");
- if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
+ if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF, EffectiveIC)) {
// Accept MaxFixedVF if we do not have a tail.
LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
return MaxFactors;
@@ -3064,7 +3065,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
auto HasOneScalarIterationRemainder =
[EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
@@ -3077,7 +3077,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 3ea49ef441def7bd2d3bb8625c66c98d82211311 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 20 May 2026 10:51:36 +0000
Subject: [PATCH 10/22] format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8308468abcd3b..972041825bd5e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3077,7 +3077,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(), EffectiveIC)) {
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue(),
+ EffectiveIC)) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
MaxFactors.ScalableVF = ElementCount::getScalable(0);
>From 95e204bf13735dd873ebf2326bc3f73d7e67aa82 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 28 May 2026 11:51:40 +0100
Subject: [PATCH 11/22] Address nit comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++-----
.../AArch64/sve-small-trip-count-vf-plus-one.ll | 10 +++++-----
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 972041825bd5e..7bb0619636e6d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3096,15 +3096,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If the maximum VF cannot produce 1 vector iteration + 1 scalar
// iteration, step down VF's to find one that can. The result should
// also eliminate any loops.
- //
- // Forced interleaving is considered when seeing if
- // OneScalarIterationRemainder is produced. It may produced more than
- // one vector iteration, but only one scalar iteration.
for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
MaxVF /= 2) {
+ // OneScalarIterationRemainder takes account of any forced
+ // interleaving.
if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
- << " with 1 scalar iteration remainder.\n");
+ << " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index d9db7d31d3b87..5c08f97a55aa2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -57,7 +57,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i16 is 8 on AArch64, so this also checks that
+; VF=8 is a natural fixed-width for i16 types on AArch64, so this also checks that
; the low-trip-count path steps down to a smaller profitable VF.
define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
@@ -107,7 +107,7 @@ exit:
}
; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
-; The natural fixed-width VF for i8 is 16 on AArch64, so this checks that the
+; VF=16 is a natural fixed-width for i8 types on AArch64, so this checks that the
; search can step down more than once before accepting VF=4.
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
@@ -264,9 +264,9 @@ exit:
ret void
}
-; ExactTC is unknown here because the loop trip count is the runtime value %n,
-; but the guard proves the maximum trip count is 5. it must not use the VF+1
-; escape because getSmallConstantTripCount returns 0.
+; In this case the vectoriser shouldn't optimise for a single vector iteration
+; + single scalar iteration, because there is no guarantee we will enter the vector
+; loop.
define void @unknown_exact_tc_max5(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
; CHECK-LABEL: define void @unknown_exact_tc_max5(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
>From 8e08177144a2647180c48c9b584e071c78885b79 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 09:54:07 +0100
Subject: [PATCH 12/22] Add cost modelling for where scalar loops are more
profitable
---
.../Vectorize/LoopVectorizationPlanner.cpp | 42 +++---
.../Vectorize/LoopVectorizationPlanner.h | 5 +
.../Transforms/Vectorize/LoopVectorize.cpp | 77 ++++++++++
.../sve-small-trip-count-vf-plus-one-cost.ll | 132 ++++++++++++++++++
4 files changed, 237 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index dbb5ad28fb4ed..f95a5c1f1a381 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -742,25 +742,8 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
if (!MaxTripCount)
return LowerCostWithoutTC;
- auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
- InstructionCost VectorCost,
- InstructionCost ScalarCost) {
- // If the trip count is a known (possibly small) constant, the trip count
- // will be rounded up to an integer number of iterations under
- // FoldTailByMasking. The total cost in that case will be
- // VecCost*ceil(TripCount/VF). When not folding the tail, the total
- // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
- // some extra overheads, but for the purpose of comparing the costs of
- // different VFs we can use this to compare the total loop-body cost
- // expected after vectorization.
- if (HasTail)
- return VectorCost * (MaxTripCount / VF) +
- ScalarCost * (MaxTripCount % VF);
- return VectorCost * divideCeil(MaxTripCount, VF);
- };
-
- auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
- auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
+ auto RTCostA = getCostForKnownTripCount(A, MaxTripCount, HasTail);
+ auto RTCostB = getCostForKnownTripCount(B, MaxTripCount, HasTail);
bool LowerCostWithTC = CmpFn(RTCostA, RTCostB);
LLVM_DEBUG(if (LowerCostWithTC != LowerCostWithoutTC) {
dbgs() << "LV: VF " << (LowerCostWithTC ? A.Width : B.Width)
@@ -782,6 +765,27 @@ bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
IsEpilogue);
}
+InstructionCost LoopVectorizationPlanner::getCostForKnownTripCount(
+ const VectorizationFactor &VF, unsigned TripCount, bool HasTail) const {
+ unsigned EstimatedWidth = VF.Width.getKnownMinValue();
+ if (std::optional<unsigned> VScale = Config.getVScaleForTuning())
+ if (VF.Width.isScalable())
+ EstimatedWidth *= *VScale;
+
+ // If the trip count is a known (possibly small) constant, the trip count
+ // will be rounded up to an integer number of iterations under
+ // FoldTailByMasking. The total cost in that case will be
+ // VecCost*ceil(TripCount/VF). When not folding the tail, the total
+ // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
+ // some extra overheads, but for the purpose of comparing the costs of
+ // different VFs we can use this to compare the total loop-body cost
+ // expected after vectorization.
+ if (HasTail)
+ return VF.Cost * (TripCount / EstimatedWidth) +
+ VF.ScalarCost * (TripCount % EstimatedWidth);
+ return VF.Cost * divideCeil(TripCount, EstimatedWidth);
+}
+
// TODO: we could return a pair of values that specify the max VF and
// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b39ffdebf6179..47b7fb86c2e26 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -1004,6 +1004,11 @@ class LoopVectorizationPlanner {
const unsigned MaxTripCount, bool HasTail,
bool IsEpilogue = false) const;
+ /// Returns the estimated loop-body cost for \p VF and a known trip count.
+ InstructionCost getCostForKnownTripCount(const VectorizationFactor &VF,
+ unsigned TripCount,
+ bool HasTail) const;
+
/// Determines if we have the infrastructure to vectorize the loop and its
/// epilogue, assuming the main loop is vectorized by \p MainPlan.
bool isCandidateForEpilogueVectorization(VPlan &MainPlan) const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7bb0619636e6d..ee55883a32f4c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5829,6 +5829,46 @@ LoopVectorizationPlanner::computeBestVF() {
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
+ bool ForceVectorization =
+ Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
+ hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned EstimatedWidth =
+ estimateElementCount(UserVF, Config.getVScaleForTuning());
+ if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
+ TC == EstimatedWidth + 1) {
+ ElementCount ScalarVF = ElementCount::getFixed(1);
+ InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ LLVM_DEBUG(dbgs()
+ << "LV: Scalar loop costs: " << ScalarCost << ".\n");
+
+ InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
+ VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
+ VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces
+ // extra control flow and a scalar epilogue for a single element, so
+ // require the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
+ AdjustedVectorCost >= ScalarCostForTC) {
+ LLVM_DEBUG(dbgs()
+ << "LV: Rejecting VF " << UserVF
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return {ScalarFactor, &FirstPlan};
+ }
+ }
+ }
+ }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -5871,6 +5911,40 @@ LoopVectorizationPlanner::computeBestVF() {
}
VPlan *PlanForBestVF = &FirstPlan;
+ ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC % EstimatedWidth != 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -5907,6 +5981,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ continue;
+
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
BestFactor = CurrentFactor;
PlanForBestVF = P.get();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
new file mode 100644
index 0000000000000..82cbb431ed76f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -0,0 +1,132 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -S %s | FileCheck %s --check-prefix=IR
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s --check-prefix=DBG
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body:
+;
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
+; DBG: LV: Using user VF 2.
+; DBG: LV: Scalar loop costs: 9.
+; DBG: Cost for VF 2: 19
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
+; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 10.
+; DBG: Cost for VF 2: 15
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
+; DBG: LV: Selecting VF: 1.
+; DBG: LV: Vectorization is possible but not beneficial.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
+ ptr noalias %c) #0 {
+; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
+; DBG-NOT: Rejecting VF 2
+; DBG: LV: Selecting VF: 2.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+declare i8 @llvm.smin.i8(i8, i8)
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.width", i32 2}
>From 73cb9bb838810711e5421b98457121199de9844d Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 29 May 2026 14:06:50 +0100
Subject: [PATCH 13/22] Remove for loop for calculating best VF
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 +++++++------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ee55883a32f4c..d989873c50c7b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3065,10 +3065,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
- auto HasOneScalarIterationRemainder =
- [EffectiveIC](ElementCount &ExactTC, unsigned int MaxVF) -> bool {
- return ExactTC.getFixedValue() == 1 + (MaxVF * EffectiveIC);
- };
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
@@ -3092,18 +3088,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// straight-line code as the both iteration counts are statically known.
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- ExactTC.getFixedValue() != 0) {
- // If the maximum VF cannot produce 1 vector iteration + 1 scalar
- // iteration, step down VF's to find one that can. The result should
- // also eliminate any loops.
- for (unsigned MaxVF = MaxFactors.FixedVF.getFixedValue(); MaxVF >= 2;
- MaxVF /= 2) {
- // OneScalarIterationRemainder takes account of any forced
- // interleaving.
- if (HasOneScalarIterationRemainder(ExactTC, MaxVF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << MaxVF
+ ExactTC.getFixedValue() > 1) {
+ unsigned TC = ExactTC.getFixedValue();
+ unsigned MaxFixedVF = MaxFactors.FixedVF.getFixedValue();
+ if ((TC - 1) % EffectiveIC == 0) {
+ unsigned VF = (TC - 1) / EffectiveIC;
+ if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
+ LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
<< " with 1 scalar iteration remaining.\n");
- MaxFactors.FixedVF = ElementCount::getFixed(MaxVF);
+ MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
return MaxFactors;
}
@@ -5924,7 +5917,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC % EstimatedWidth != 1)
+ if (TC != EstimatedWidth + 1)
return false;
InstructionCost VectorCost =
>From e6129ebc268253494ba1709bf9790d227d4455da Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 23 Jun 2026 09:30:56 +0100
Subject: [PATCH 14/22] Refactor IsUnprofitableOneScalarTail Lambda Function
---
.../Transforms/Vectorize/LoopVectorize.cpp | 92 +++++++++----------
1 file changed, 42 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d989873c50c7b..c9af15e7e08b4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5814,6 +5814,42 @@ LoopVectorizationPlanner::computeBestVF() {
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
+ auto IsUnprofitableOneScalarTail =
+ [&](const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ const InstructionCost &ScalarCost) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != EstimatedWidth + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, HasTail);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
+ if (!AdjustedVectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+ };
ElementCount UserVF = Hints.getWidth();
if (VPlans.size() == 1) {
@@ -5840,23 +5876,10 @@ LoopVectorizationPlanner::computeBestVF() {
InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(UserFactor, TC, /*HasTail=*/true);
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces
- // extra control flow and a scalar epilogue for a single element, so
- // require the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (VectorCost.isValid() && ScalarCostForTC.isValid() &&
- AdjustedVectorCost >= ScalarCostForTC) {
- LLVM_DEBUG(dbgs()
- << "LV: Rejecting VF " << UserVF
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
+ if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
+ ForceVectorization, ExactTC,
+ ScalarFactor, ScalarCost)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -5905,39 +5928,6 @@ LoopVectorizationPlanner::computeBestVF() {
VPlan *PlanForBestVF = &FirstPlan;
ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!VectorCost.isValid() || !ScalarCostForTC.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
for (auto &P : VPlans) {
ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
@@ -5974,7 +5964,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail()))
+ if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor,
+ ScalarCost))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
>From 54445e18410f15bee2bb05047f6e36e961fc09b8 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 2 Jul 2026 15:54:45 +0100
Subject: [PATCH 15/22] Add consideration of IC and test
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 ++--
.../sve-small-trip-count-vf-plus-one-cost.ll | 46 ++++++++++++++++---
2 files changed, 44 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c9af15e7e08b4..7c63c94e70df7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5818,7 +5818,7 @@ LoopVectorizationPlanner::computeBestVF() {
[&](const VectorizationFactor &CurrentFactor, bool HasTail,
bool ForceVectorization, const ElementCount &ExactTC,
const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost) {
+ const InstructionCost &ScalarCost, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5829,7 +5829,7 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned EstimatedWidth = estimateElementCount(
CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != EstimatedWidth + 1)
+ if (TC != (EstimatedWidth * UserIC) + 1)
return false;
InstructionCost VectorCost =
@@ -5852,6 +5852,7 @@ LoopVectorizationPlanner::computeBestVF() {
};
ElementCount UserVF = Hints.getWidth();
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
@@ -5879,7 +5880,7 @@ LoopVectorizationPlanner::computeBestVF() {
VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost)) {
+ ScalarFactor, ScalarCost, UserIC)) {
return {ScalarFactor, &FirstPlan};
}
}
@@ -5966,7 +5967,7 @@ LoopVectorizationPlanner::computeBestVF() {
if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost))
+ ScalarCost, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 82cbb431ed76f..70eb5b07c25aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -7,13 +7,7 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc3_udiv_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
-; IR-LABEL: define void @tc3_udiv_i8_forced(
-; IR: vector.body:
+; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
@@ -44,6 +38,9 @@ exit:
define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
; DBG: LV: Using user VF 2.
; DBG: LV: Scalar loop costs: 9.
@@ -71,6 +68,9 @@ exit:
}
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; IR-LABEL: define void @tc3_smin_i8_reject(
+; IR-NOT: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
@@ -99,6 +99,9 @@ exit:
define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
+; IR-LABEL: define void @tc3_udiv_i8_forced(
+; IR: vector.body
+
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_forced'
; DBG-NOT: Rejecting VF 2
; DBG: LV: Selecting VF: 2.
@@ -122,6 +125,33 @@ exit:
ret void
}
+define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias %c) #0{
+; IR-LABEL: define void @tc5_udiv_i8_reject_ic2(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
+; DBG-NOT: LV: Selecting VF: 2.
+; DBG Rejecting VF 2
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+
+exit:
+ ret void
+}
+
declare i8 @llvm.smin.i8(i8, i8)
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
@@ -130,3 +160,5 @@ attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.vectorize.width", i32 2}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.interleave.count", i32 2}
>From aaf7100ce99909116a5f9c12b0b3a764863781a2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:25:20 +0100
Subject: [PATCH 16/22] Respond to review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 7 ++
.../Transforms/Vectorize/LoopVectorize.cpp | 109 +++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 75 ++++++------
.../sve-small-trip-count-vf-plus-one.ll | 81 +++++++++++++
4 files changed, 166 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 47b7fb86c2e26..6801dfcd9c64f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -878,6 +878,13 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
+ /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
+ /// it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
/// ProfitableVFs.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7c63c94e70df7..786c238009290 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3094,7 +3094,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if ((TC - 1) % EffectiveIC == 0) {
unsigned VF = (TC - 1) / EffectiveIC;
if (VF >= 2 && VF <= MaxFixedVF && isPowerOf2_32(VF)) {
- LLVM_DEBUG(dbgs() << "LV: Picking VF=" << VF
+ LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
<< " with 1 scalar iteration remaining.\n");
MaxFactors.FixedVF = ElementCount::getFixed(VF);
MaxFactors.ScalableVF = ElementCount::getScalable(0);
@@ -5808,84 +5808,57 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+ if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
+ CurrentFactor.Width.isScalable())
+ return false;
+
+ unsigned TC = ExactTC.getFixedValue();
+ if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
+ return false;
+
+ unsigned EstimatedWidth = estimateElementCount(
+ CurrentFactor.Width, Config.getVScaleForTuning());
+ if (TC != (EstimatedWidth * UserIC) + 1)
+ return false;
+
+ InstructionCost VectorCost =
+ getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
+ InstructionCost ScalarCostForTC =
+ getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
+ // Be conservative for the one-scalar-tail shape. It introduces extra
+ // control flow and a scalar epilogue for a single element, so require
+ // the vectorized form to save at least one scalar iteration.
+ InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
+ if (!VectorCost.isValid() ||
+ AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " >= scalar cost "
+ << ScalarCostForTC << ".\n");
+ return true;
+}
+
std::pair<VectorizationFactor, VPlan *>
LoopVectorizationPlanner::computeBestVF() {
if (VPlans.empty())
return {VectorizationFactor::Disabled(), nullptr};
// If there is a single VPlan with a single VF, return it directly.
VPlan &FirstPlan = *VPlans[0];
- auto IsUnprofitableOneScalarTail =
- [&](const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor,
- const InstructionCost &ScalarCost, unsigned int UserIC) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
-
- unsigned TC = ExactTC.getFixedValue();
- if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
-
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != (EstimatedWidth * UserIC) + 1)
- return false;
-
- InstructionCost VectorCost =
- getCostForKnownTripCount(CurrentFactor, TC, HasTail);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarCost;
- if (!AdjustedVectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC)
- return false;
-
- LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
- << " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
- };
ElementCount UserVF = Hints.getWidth();
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (VPlans.size() == 1) {
// For outer loops, the plan has a single vector VF determined by the
// heuristic.
assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
FirstPlan.isOuterLoop()) &&
"must have a single scalar VF, UserVF or an outer loop");
- bool ForceVectorization =
- Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- if (!FirstPlan.hasScalarVFOnly() && !FirstPlan.isOuterLoop() &&
- hasPlanWithVF(UserVF) && UserVF.isVector() && !ForceVectorization) {
- ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- if (FirstPlan.hasScalarTail() && ExactTC.isFixed() && UserVF.isFixed()) {
- unsigned TC = ExactTC.getFixedValue();
- unsigned EstimatedWidth =
- estimateElementCount(UserVF, Config.getVScaleForTuning());
- if (TC != 0 && TC <= TTI.getMinTripCountTailFoldingThreshold() &&
- TC == EstimatedWidth + 1) {
- ElementCount ScalarVF = ElementCount::getFixed(1);
- InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
- LLVM_DEBUG(dbgs()
- << "LV: Scalar loop costs: " << ScalarCost << ".\n");
-
- InstructionCost Cost = cost(FirstPlan, UserVF, /*RU=*/nullptr);
- VectorizationFactor UserFactor(UserVF, Cost, ScalarCost);
- VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
- if (IsUnprofitableOneScalarTail(UserFactor, FirstPlan.hasScalarTail(),
- ForceVectorization, ExactTC,
- ScalarFactor, ScalarCost, UserIC)) {
- return {ScalarFactor, &FirstPlan};
- }
- }
- }
- }
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
@@ -5965,9 +5938,9 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (IsUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor,
- ScalarCost, UserIC))
+ unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
+ ForceVectorization, ExactTC, ScalarFactor, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 70eb5b07c25aa..4516062f9b7d1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -10,7 +10,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; IR-NOT: vector.body:
;
; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
@@ -36,43 +36,12 @@ exit:
ret void
}
-define void @tc3_udiv_i8_user_vf2(ptr noalias %a, ptr noalias %b,
- ptr noalias %c) #0 {
-; IR-LABEL: define void @tc3_udiv_i8_user_vf2(
-; IR-NOT: vector.body
-
-; DBG-LABEL: LV: Checking a loop in 'tc3_udiv_i8_user_vf2'
-; DBG: LV: Using user VF 2.
-; DBG: LV: Scalar loop costs: 9.
-; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
-; DBG: LV: Vectorization is possible but not beneficial.
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %pa = getelementptr inbounds i8, ptr %a, i64 %iv
- %pb = getelementptr inbounds i8, ptr %b, i64 %iv
- %pc = getelementptr inbounds i8, ptr %c, i64 %iv
- %va = load i8, ptr %pa, align 1
- %vb = load i8, ptr %pb, align 1
- %div = udiv i8 %va, %vb
- store i8 %div, ptr %pc, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
-
-exit:
- ret void
-}
-
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
; IR-NOT: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
-; DBG: LV: Picking VF=2 with 1 scalar iteration remaining.
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
@@ -131,7 +100,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG Rejecting VF 2
+; DBG: Rejecting VF 2
entry:
br label %loop
@@ -146,7 +115,37 @@ loop:
store i8 %div, ptr %pc, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 5
- br i1 %exitcond, label %exit, label %loop, !llvm.loop !4
+ br i1 %exitcond, label %exit, label %loop, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
+ ptr noalias %c) #0 {
+; IR-LABEL: define void @tc5_sin_f32_select_smaller_vf(
+; IR: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
+; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
+; DBG: LV: Rejecting VF 4 for one-scalar-tail low trip count: vector cost 88 >= scalar cost 80.
+; DBG-NOT: Selecting VF: 4
+; DBG: LV: Selecting VF: 2.
+
+
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds float, ptr %a, i64 %iv
+ %pc = getelementptr inbounds float, ptr %c, i64 %iv
+ %va = load float, ptr %pa, align 4
+ %sin = tail call float @llvm.sin.f32(float %va)
+ store float %sin, ptr %pc, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
exit:
ret void
@@ -154,11 +153,11 @@ exit:
declare i8 @llvm.smin.i8(i8, i8)
+declare float @llvm.sin.f32(float)
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = distinct !{!2, !3}
-!3 = !{!"llvm.loop.vectorize.width", i32 2}
-!4 = distinct !{!4, !5}
-!5 = !{!"llvm.loop.interleave.count", i32 2}
+!3 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 5c08f97a55aa2..b999acc21309c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -375,6 +375,87 @@ exit:
ret void
}
+; For this example, vectorization should not take place as the
+; scalar cost is better than the vectorized cost.
+define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
+; CHECK-LABEL: define void @tc3_udiv_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[PB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[PC:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT: [[VA:%.*]] = load i8, ptr [[PA]], align 1
+; CHECK-NEXT: [[VB:%.*]] = load i8, ptr [[PB]], align 1
+; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[VA]], [[VB]]
+; CHECK-NEXT: store i8 [[DIV]], ptr [[PC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+ ptr noalias %c) #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i8, ptr %a, i64 %iv
+ %pb = getelementptr inbounds i8, ptr %b, i64 %iv
+ %pc = getelementptr inbounds i8, ptr %c, i64 %iv
+ %va = load i8, ptr %pa, align 1
+ %vb = load i8, ptr %pb, align 1
+ %div = udiv i8 %va, %vb
+ store i8 %div, ptr %pc, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_smin_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[TMP1]], i8 [[TMP2]])
+; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
+ store i8 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From ecf68082ad3a46259c3dada8e0a5b03d312a74f2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 16 Jul 2026 15:55:34 +0100
Subject: [PATCH 17/22] format
---
.../Vectorize/LoopVectorizationPlanner.h | 14 +++++----
.../Transforms/Vectorize/LoopVectorize.cpp | 29 +++++++++++--------
2 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 6801dfcd9c64f..127664101a839 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -878,12 +878,14 @@ class LoopVectorizationPlanner {
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
- /// Examines if it is unprofitable to Vectorize a small loop in a way that leaves a
- /// Vector iteration, followed by a single iteration scalar tail. For some uses cases,
- /// it is better to leave the original Scalar loop in place.
- bool isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC);
+ /// Examines if it is unprofitable to Vectorize a small loop in a way that
+ /// leaves a Vector iteration, followed by a single iteration scalar tail. For
+ /// some uses cases, it is better to leave the original Scalar loop in place.
+ bool isUnprofitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ bool HasTail, bool ForceVectorization,
+ const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 786c238009290..a269c01a43091 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5808,9 +5808,10 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
+bool LoopVectorizationPlanner::isUnprofitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, bool HasTail,
+ bool ForceVectorization, const ElementCount &ExactTC,
+ const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
CurrentFactor.Width.isScalable())
return false;
@@ -5819,8 +5820,8 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
return false;
- unsigned EstimatedWidth = estimateElementCount(
- CurrentFactor.Width, Config.getVScaleForTuning());
+ unsigned EstimatedWidth =
+ estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
if (TC != (EstimatedWidth * UserIC) + 1)
return false;
@@ -5832,11 +5833,13 @@ bool LoopVectorizationPlanner::isUnprofitableOneScalarTail (const VectorizationF
// control flow and a scalar epilogue for a single element, so require
// the vectorized form to save at least one scalar iteration.
InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
- if (!VectorCost.isValid() ||
- AdjustedVectorCost < ScalarCostForTC) {
- LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width << " for one-scalar-tail low trip count: vector cost " << AdjustedVectorCost << " < " << ScalarCostForTC << ".\n");
- return false;
- }
+ if (!VectorCost.isValid() || AdjustedVectorCost < ScalarCostForTC) {
+ LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count: vector cost "
+ << AdjustedVectorCost << " < " << ScalarCostForTC
+ << ".\n");
+ return false;
+ }
LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
@@ -5938,9 +5941,11 @@ LoopVectorizationPlanner::computeBestVF() {
cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- unsigned int UserIC = Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
+ unsigned int UserIC =
+ Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor, UserIC))
+ ForceVectorization, ExactTC, ScalarFactor,
+ UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
>From 9e06c389d8f99e0200d3b5e669f99ed8add49800 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 20 Jul 2026 10:04:12 +0100
Subject: [PATCH 18/22] Respond to review comments
---
.../Vectorize/LoopVectorizationPlanner.h | 13 +++---
.../Transforms/Vectorize/LoopVectorize.cpp | 42 ++++++++-----------
.../sve-small-trip-count-vf-plus-one-cost.ll | 8 ++--
.../sve-small-trip-count-vf-plus-one.ll | 24 ++++++-----
4 files changed, 42 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a4f57c6a02570..b9c975ba8a046 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -901,13 +901,12 @@ class LoopVectorizationPlanner {
VPlan &getPlanFor(ElementCount VF) const;
/// Examines if it is unprofitable to Vectorize a small loop in a way that
- /// leaves a Vector iteration, followed by a single iteration scalar tail. For
- /// some uses cases, it is better to leave the original Scalar loop in place.
- bool isUnprofitableOneScalarTail(const VectorizationFactor &CurrentFactor,
- bool HasTail, bool ForceVectorization,
- const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor,
- unsigned int UserIC);
+ /// leaves a Vector iteration, followed by a single iteration scalar tail.
+ /// Returns true if it is profitable to vectorize a small loop with a single
+ /// scalar tail iteration
+ bool isProfitableOneScalarTail(const VectorizationFactor &CurrentFactor,
+ const ElementCount &ExactTC,
+ unsigned int UserIC);
/// Compute and return the most profitable vectorization factor and the
/// corresponding best VPlan. Also collect all profitable VFs in
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2e775671aa9a4..5c43c9b272d42 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5810,44 +5810,39 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
return Cost;
}
-bool LoopVectorizationPlanner::isUnprofitableOneScalarTail(
- const VectorizationFactor &CurrentFactor, bool HasTail,
- bool ForceVectorization, const ElementCount &ExactTC,
- const VectorizationFactor &ScalarFactor, unsigned int UserIC) {
- if (ForceVectorization || !HasTail || !ExactTC.isFixed() ||
- CurrentFactor.Width.isScalable())
- return false;
+bool LoopVectorizationPlanner::isProfitableOneScalarTail(
+ const VectorizationFactor &CurrentFactor, const ElementCount &ExactTC,
+ unsigned int UserIC) {
+ if (!ExactTC.isFixed() || CurrentFactor.Width.isScalable())
+ return true;
unsigned TC = ExactTC.getFixedValue();
if (TC == 0 || TC > TTI.getMinTripCountTailFoldingThreshold())
- return false;
+ return true;
unsigned EstimatedWidth =
estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
if (TC != (EstimatedWidth * UserIC) + 1)
- return false;
+ return true;
+ // VectorCost reflects the cost of the requried vector iteration(s) and the
+ // one remaining scalar iteration cost
InstructionCost VectorCost =
getCostForKnownTripCount(CurrentFactor, TC, /*HasTail=*/true);
- InstructionCost ScalarCostForTC =
- getCostForKnownTripCount(ScalarFactor, TC, /*HasTail=*/false);
- // Be conservative for the one-scalar-tail shape. It introduces extra
- // control flow and a scalar epilogue for a single element, so require
- // the vectorized form to save at least one scalar iteration.
- InstructionCost AdjustedVectorCost = VectorCost + ScalarFactor.ScalarCost;
- if (!VectorCost.isValid() || AdjustedVectorCost < ScalarCostForTC) {
+ InstructionCost ScalarCostForTC = CurrentFactor.ScalarCost * TC;
+ if (!VectorCost.isValid() || VectorCost < ScalarCostForTC) {
LLVM_DEBUG(dbgs() << "LV: Accepting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " < " << ScalarCostForTC
+ << VectorCost << " < scalar cost " << ScalarCostForTC
<< ".\n");
- return false;
+ return true;
}
LLVM_DEBUG(dbgs() << "LV: Rejecting VF " << CurrentFactor.Width
<< " for one-scalar-tail low trip count: vector cost "
- << AdjustedVectorCost << " >= scalar cost "
- << ScalarCostForTC << ".\n");
- return true;
+ << VectorCost << " >= scalar cost " << ScalarCostForTC
+ << ".\n");
+ return false;
}
std::pair<VectorizationFactor, VPlan *>
@@ -5944,9 +5939,8 @@ LoopVectorizationPlanner::computeBestVF() {
unsigned int UserIC =
Hints.getInterleave() != 0 ? Hints.getInterleave() : 1;
- if (isUnprofitableOneScalarTail(CurrentFactor, P->hasScalarTail(),
- ForceVectorization, ExactTC, ScalarFactor,
- UserIC))
+ if (!ForceVectorization && P->hasScalarTail() &&
+ !isProfitableOneScalarTail(CurrentFactor, ExactTC, UserIC))
continue;
if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 4516062f9b7d1..e877af9dc88e8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -13,7 +13,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 37 >= scalar cost 27.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 27.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -44,7 +44,7 @@ define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 35 >= scalar cost 30.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 40 >= scalar cost 30.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -100,7 +100,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG: Rejecting VF 2
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 85 >= scalar cost 45.
entry:
br label %loop
@@ -128,7 +128,7 @@ define void @tc5_sin_f32_select_smaller_vf(ptr noalias %a,
; DBG-LABEL: LV: Checking a loop in 'tc5_sin_f32_select_smaller_vf'
; DBG: Picking MaxVF=4 with 1 scalar iteration remaining.
-; DBG: LV: Rejecting VF 4 for one-scalar-tail low trip count: vector cost 88 >= scalar cost 80.
+; DBG: LV: Accepting VF 4 for one-scalar-tail low trip count: vector cost 72 < scalar cost 80.
; DBG-NOT: Selecting VF: 4
; DBG: LV: Selecting VF: 2.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index b999acc21309c..dc947ff778280 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -106,9 +106,8 @@ exit:
ret void
}
-; TC=5, VF=4: TC == VF + 1 (5 == 4 + 1).
; VF=16 is a natural fixed-width for i8 types on AArch64, so this checks that the
-; search can step down more than once before accepting VF=4.
+; search can step down more than once before accepting the most acceptable VF.
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
@@ -117,10 +116,15 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -134,7 +138,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -188,7 +192,7 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -242,7 +246,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -351,7 +355,7 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 1d7d89b475afc20fb1570ed87e7e6684d9d95ec6 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Tue, 21 Jul 2026 13:23:18 +0100
Subject: [PATCH 19/22] Fix test failures
---
.../sve-small-trip-count-vf-plus-one-cost.ll | 14 +-
.../sve-small-trip-count-vf-plus-one.ll | 142 ++++++++++--------
2 files changed, 84 insertions(+), 72 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index e877af9dc88e8..5c7da2b49df27 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -13,7 +13,7 @@ define void @tc3_udiv_i8_reject(ptr noalias %a, ptr noalias %b,
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 9.
; DBG: Cost for VF 2: 19
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 27.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 28 >= scalar cost 27.
; DBG: LV: Selecting VF: 1.
; DBG: LV: Vectorization is possible but not beneficial.
entry:
@@ -36,17 +36,19 @@ exit:
ret void
}
+; FIXME: This is currently accepted as cost for vector is smaller than scalar.
+; Performance is poor due to poor CodeGen of using type promotion instead of
+; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
+; IR: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
; DBG: Cost for VF 2: 15
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 40 >= scalar cost 30.
-; DBG: LV: Selecting VF: 1.
-; DBG: LV: Vectorization is possible but not beneficial.
+; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 25 < scalar cost 30.
+; DBG: LV: Selecting VF: 2.
entry:
br label %loop
@@ -100,7 +102,7 @@ define void @tc5_udiv_i8_reject_ic2(ptr noalias %a, ptr noalias %b, ptr noalias
; DBG-LABEL: LV: Checking a loop in 'tc5_udiv_i8_reject_ic2'
; DBG-NOT: LV: Selecting VF: 2.
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 85 >= scalar cost 45.
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 47 >= scalar cost 45.
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index dc947ff778280..f26f1fdc7ccce 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -12,9 +12,9 @@ target triple = "aarch64-unknown-linux-gnu"
define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[SCALAR_PH1:.*:]]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
@@ -22,11 +22,11 @@ define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[B]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -34,7 +34,7 @@ define void @tc5_vf4_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -62,9 +62,9 @@ exit:
define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i16(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[A]], align 2
@@ -72,11 +72,11 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <4 x i16> [[TMP0]], ptr [[B]], align 2
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i16, ptr [[GEP_A]], align 2
@@ -84,7 +84,7 @@ define void @tc5_vf4_vectorize_i16(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_B]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -111,26 +111,21 @@ exit:
define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_vectorize_i8(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: store <4 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[GEP_A]], align 1
@@ -138,7 +133,7 @@ define void @tc5_vf4_vectorize_i8(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_B]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -165,9 +160,9 @@ exit:
define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH1:.*:]]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
@@ -180,11 +175,11 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_A]], align 4
@@ -192,7 +187,7 @@ define void @tc5_forced_ic2_vectorize_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -219,9 +214,9 @@ exit:
define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_forced_ic2_vectorize_i64(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*:]]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
@@ -234,11 +229,11 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i64, ptr [[GEP_A]], align 4
@@ -246,7 +241,7 @@ define void @tc5_forced_ic2_vectorize_i64(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_B]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -327,9 +322,9 @@ exit:
define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc5_vf4_unsafe_useric_distance4_i32(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 4
@@ -341,11 +336,11 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP0]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 8, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[B_DST:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[B_SRC:%.*]] = getelementptr inbounds i32, ptr [[B_DST]], i64 -4
; CHECK-NEXT: [[DEP:%.*]] = load i32, ptr [[B_SRC]], align 4
@@ -355,7 +350,7 @@ define void @tc5_vf4_unsafe_useric_distance4_i32(ptr noalias %a, ptr noalias %b)
; CHECK-NEXT: store i32 [[ADD]], ptr [[B_DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 9
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -422,13 +417,28 @@ exit:
ret void
}
+; FIXME: This is currently accepted as cost for vector is smaller than scalar.
+; Performance is poor due to poor CodeGen of using type promotion instead of
+; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc3_smin_i8_reject(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[B]], align 1
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i8>, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[WIDE_LOAD]], <2 x i8> [[WIDE_LOAD1]])
+; CHECK-NEXT: store <2 x i8> [[TMP0]], ptr [[B]], align 1
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
@@ -437,7 +447,7 @@ define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 4853466d83a1f7cb881d620b5bef41d223fed27f Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 27 Jul 2026 09:46:28 +0100
Subject: [PATCH 20/22] Adjust cost for min/max intrinsics using v2i8 and v2i16
---
.../AArch64/AArch64TargetTransformInfo.cpp | 6 ++
.../Analysis/CostModel/AArch64/min-max.ll | 8 +--
.../LoopVectorize/AArch64/intrinsiccost.ll | 2 +-
.../sve-small-trip-count-vf-plus-one-cost.ll | 40 ++++++++++---
.../sve-small-trip-count-vf-plus-one.ll | 59 +++++++++++++------
5 files changed, 85 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 03aa42b31e6aa..060976ab501e2 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -711,6 +711,12 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
MVT::nxv2i64};
auto LT = getTypeLegalizationCost(RetTy);
+ // Type promotion for v2i8 and v2i16 types have a heavy cost when vectorising.
+ // Account for this cost to avoid vectorising unprofitable examples when vectorising
+ // loops with low trip counts.
+ MVT VT = MVT::getVT(RetTy);
+ if (VT == MVT::v2i8 || VT == MVT::v2i16)
+ return LT.first * 6;
// v2i64 types get converted to cmp+bif hence the cost of 2
if (LT.second == MVT::v2i64)
return LT.first * 2;
diff --git a/llvm/test/Analysis/CostModel/AArch64/min-max.ll b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
index a579eb37ff22a..b4bef140e65e1 100644
--- a/llvm/test/Analysis/CostModel/AArch64/min-max.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
@@ -17,7 +17,7 @@ define void @umin() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -64,7 +64,7 @@ define void @umax() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -111,7 +111,7 @@ define void @smin() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -158,7 +158,7 @@ define void @smax() {
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
index c8555a8629ec5..c2f5ca4e0538f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
@@ -128,7 +128,7 @@ while.end:
; CHECK-COST-LABEL: umin
; CHECK-COST: Found an estimated cost of 2 for VF 1 For instruction: %1 = tail call i8 @llvm.umin.i8(i8 %0, i8 %offset)
-; CHECK-COST: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
+; CHECK-COST: Cost of 6 for VF 2: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 8: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 16: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 5c7da2b49df27..1e7801f89fda8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -36,19 +36,16 @@ exit:
ret void
}
-; FIXME: This is currently accepted as cost for vector is smaller than scalar.
-; Performance is poor due to poor CodeGen of using type promotion instead of
-; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR: vector.body
+; IR-NOT: vector.body
; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
; DBG: LV: Scalar loop costs: 10.
-; DBG: Cost for VF 2: 15
-; DBG: LV: Accepting VF 2 for one-scalar-tail low trip count: vector cost 25 < scalar cost 30.
-; DBG: LV: Selecting VF: 2.
+; DBG: Cost for VF 2: 20
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 30 >= scalar cost 30.
+; DBG-NOT: LV: Selecting VF: 2.
entry:
br label %loop
@@ -68,6 +65,35 @@ exit:
ret void
}
+define void @tc3_smin_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
+; IR-LABEL: define void @tc3_smin_i16_reject(
+; IR-NOT: vector.body
+
+; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i16_reject'
+; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
+; DBG: LV: Scalar loop costs: 10.
+; DBG: Cost for VF 2: 20
+; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 30 >= scalar cost 30.
+; DBG-NOT: LV: Selecting VF: 2.
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i16, ptr %b, i64 %iv
+ %0 = load i16, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %iv
+ %1 = load i16, ptr %arrayidx2, align 1
+ %min = tail call i16 @llvm.smin.i16(i16 %0, i16 %1)
+ store i16 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc3_udiv_i8_forced(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index f26f1fdc7ccce..8a593fa8a8063 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -417,28 +417,13 @@ exit:
ret void
}
-; FIXME: This is currently accepted as cost for vector is smaller than scalar.
-; Performance is poor due to poor CodeGen of using type promotion instead of
-; type widening.
define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-LABEL: define void @tc3_smin_i8_reject(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[B]], align 1
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i8>, ptr [[A]], align 1
-; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[WIDE_LOAD]], <2 x i8> [[WIDE_LOAD1]])
-; CHECK-NEXT: store <2 x i8> [[TMP0]], ptr [[B]], align 1
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[SCALAR_PH1:.*]]
-; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
@@ -447,7 +432,7 @@ define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -470,6 +455,44 @@ exit:
ret void
}
+define void @tc3_smin_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_smin_i16_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i16 @llvm.smin.i16(i16 [[TMP1]], i16 [[TMP2]])
+; CHECK-NEXT: store i16 [[MIN]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i16, ptr %b, i64 %iv
+ %0 = load i16, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %iv
+ %1 = load i16, ptr %arrayidx2, align 1
+ %min = tail call i16 @llvm.smin.i16(i16 %0, i16 %1)
+ store i16 %min, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From a0a06dd9df8531cc83e0fa1ee5810e67f32f7912 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 27 Jul 2026 09:56:15 +0100
Subject: [PATCH 21/22] format
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 060976ab501e2..b1847ba81c36d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -711,9 +711,9 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
MVT::nxv2i64};
auto LT = getTypeLegalizationCost(RetTy);
- // Type promotion for v2i8 and v2i16 types have a heavy cost when vectorising.
- // Account for this cost to avoid vectorising unprofitable examples when vectorising
- // loops with low trip counts.
+ // Type promotion for v2i8 and v2i16 types have a heavy cost when
+ // vectorising. Account for this cost to avoid vectorising unprofitable
+ // examples when vectorising loops with low trip counts.
MVT VT = MVT::getVT(RetTy);
if (VT == MVT::v2i8 || VT == MVT::v2i16)
return LT.first * 6;
>From c1e95a86c849434d05ed8d9f74064d22925445c8 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 29 Jul 2026 10:18:07 +0100
Subject: [PATCH 22/22] Replace cost model changes for min/max with targeted
rejection
For Min/Max intrinsics and XOr operations, v2i8, v4i8 and v2i16 are
not desirable to vectorise due to type promotion CodeGeReplace cost
model changes for min/max with targeted rejection. These are left
as the original scalar loops
---
.../AArch64/AArch64TargetTransformInfo.cpp | 6 -
.../Transforms/Vectorize/LoopVectorize.cpp | 42 ++++-
.../Analysis/CostModel/AArch64/min-max.ll | 16 +-
.../LoopVectorize/AArch64/intrinsiccost.ll | 2 +-
.../sve-small-trip-count-vf-plus-one-cost.ll | 58 ------
.../sve-small-trip-count-vf-plus-one.ll | 166 +++++++++++++++++-
6 files changed, 208 insertions(+), 82 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b1847ba81c36d..03aa42b31e6aa 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -711,12 +711,6 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
MVT::nxv2i64};
auto LT = getTypeLegalizationCost(RetTy);
- // Type promotion for v2i8 and v2i16 types have a heavy cost when
- // vectorising. Account for this cost to avoid vectorising unprofitable
- // examples when vectorising loops with low trip counts.
- MVT VT = MVT::getVT(RetTy);
- if (VT == MVT::v2i8 || VT == MVT::v2i16)
- return LT.first * 6;
// v2i64 types get converted to cmp+bif hence the cost of 2
if (LT.second == MVT::v2i64)
return LT.first * 2;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7a8720b9ded83..a306f3b48c2f0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5842,9 +5842,49 @@ bool LoopVectorizationPlanner::isProfitableOneScalarTail(
unsigned EstimatedWidth =
estimateElementCount(CurrentFactor.Width, Config.getVScaleForTuning());
- if (TC != (EstimatedWidth * UserIC) + 1)
+ if (TC % (EstimatedWidth * UserIC) != 1)
return true;
+ // On certain Instructions or Intrinsics, where Type Promotion is used
+ // for v2i8, v4i8 and v2i16 types for legalization. These are not
+ // beneficial for Vectorization due to CodeGen of Type Promotion so
+ // the Scalar loop is preffered.
+ for (BasicBlock *BB : OrigLoop->blocks()) {
+ for (Instruction &Inst : *BB) {
+ FixedVectorType *VTy =
+ dyn_cast<FixedVectorType>(toVectorTy(Inst.getType(), EstimatedWidth));
+ if (!VTy)
+ continue;
+
+ bool IsUndesirableType =
+ (VTy->getScalarSizeInBits() == 8 &&
+ (EstimatedWidth * UserIC == 2 || EstimatedWidth * UserIC == 4)) ||
+ (VTy->getScalarSizeInBits() == 16 && EstimatedWidth * UserIC == 2);
+ if (auto Opcode = Inst.getOpcode();
+ Opcode == Instruction::Xor && IsUndesirableType) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count. Vectorizing "
+ "with Type Promotion is unprofitable for Xor operations.\n");
+ return false;
+ }
+ if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
+ Intrinsic::ID IID = II->getIntrinsicID();
+ if ((IID == Intrinsic::smin || IID == Intrinsic::smax ||
+ IID == Intrinsic::umin || IID == Intrinsic::umax) &&
+ IsUndesirableType) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Rejecting VF " << CurrentFactor.Width
+ << " for one-scalar-tail low trip count. Vectorizing with "
+ "Type Promotion is unprofitable for Min/Max intrinsics.\n");
+ return false;
+ }
+ }
+ }
+ }
+
// VectorCost reflects the cost of the requried vector iteration(s) and the
// one remaining scalar iteration cost
InstructionCost VectorCost =
diff --git a/llvm/test/Analysis/CostModel/AArch64/min-max.ll b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
index e04898634ae6f..5e5026ccfa4df 100644
--- a/llvm/test/Analysis/CostModel/AArch64/min-max.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
@@ -84,7 +84,7 @@ define void @umin() {
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-BASE-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -104,7 +104,7 @@ define void @umin() {
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-CSSC-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -145,7 +145,7 @@ define void @umax() {
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-BASE-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -165,7 +165,7 @@ define void @umax() {
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-CSSC-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -206,7 +206,7 @@ define void @smin() {
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-BASE-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -226,7 +226,7 @@ define void @smin() {
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-CSSC-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -267,7 +267,7 @@ define void @smax() {
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-BASE-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-BASE-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -287,7 +287,7 @@ define void @smax() {
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-CSSC-NEXT: Cost Model: Found costs of 6 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
; CHECK-CSSC-NEXT: Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
index c2f5ca4e0538f..c8555a8629ec5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll
@@ -128,7 +128,7 @@ while.end:
; CHECK-COST-LABEL: umin
; CHECK-COST: Found an estimated cost of 2 for VF 1 For instruction: %1 = tail call i8 @llvm.umin.i8(i8 %0, i8 %offset)
-; CHECK-COST: Cost of 6 for VF 2: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
+; CHECK-COST: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 8: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
; CHECK-COST: Cost of 1 for VF 16: WIDEN-INTRINSIC ir<%1> = call llvm.umin(ir<%0>, ir<%offset>)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
index 1e7801f89fda8..6faefb336658f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one-cost.ll
@@ -36,64 +36,6 @@ exit:
ret void
}
-define void @tc3_smin_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
-; IR-LABEL: define void @tc3_smin_i8_reject(
-; IR-NOT: vector.body
-
-; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i8_reject'
-; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
-; DBG: LV: Scalar loop costs: 10.
-; DBG: Cost for VF 2: 20
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 30 >= scalar cost 30.
-; DBG-NOT: LV: Selecting VF: 2.
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
- %0 = load i8, ptr %arrayidx, align 1
- %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
- %1 = load i8, ptr %arrayidx2, align 1
- %min = tail call i8 @llvm.smin.i8(i8 %0, i8 %1)
- store i8 %min, ptr %arrayidx, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- ret void
-}
-
-define void @tc3_smin_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
-; IR-LABEL: define void @tc3_smin_i16_reject(
-; IR-NOT: vector.body
-
-; DBG-LABEL: LV: Checking a loop in 'tc3_smin_i16_reject'
-; DBG: LV: Picking MaxVF=2 with 1 scalar iteration remaining.
-; DBG: LV: Scalar loop costs: 10.
-; DBG: Cost for VF 2: 20
-; DBG: LV: Rejecting VF 2 for one-scalar-tail low trip count: vector cost 30 >= scalar cost 30.
-; DBG-NOT: LV: Selecting VF: 2.
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %arrayidx = getelementptr inbounds i16, ptr %b, i64 %iv
- %0 = load i16, ptr %arrayidx, align 1
- %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %iv
- %1 = load i16, ptr %arrayidx2, align 1
- %min = tail call i16 @llvm.smin.i16(i16 %0, i16 %1)
- store i16 %min, ptr %arrayidx, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, 3
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- ret void
-}
-
define void @tc3_udiv_i8_forced(ptr noalias %a, ptr noalias %b,
ptr noalias %c) #0 {
; IR-LABEL: define void @tc3_udiv_i8_forced(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
index 8a593fa8a8063..84a676edd8df0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-small-trip-count-vf-plus-one.ll
@@ -455,18 +455,56 @@ exit:
ret void
}
-define void @tc3_smin_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
-; CHECK-LABEL: define void @tc3_smin_i16_reject(
+define void @tc5_smax_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_smax_i8_reject(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i8 @llvm.smax.i8(i8 [[TMP0]], i8 [[TMP1]])
+; CHECK-NEXT: store i8 [[MIN]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %max = tail call i8 @llvm.smax.i8(i8 %0, i8 %1)
+ store i8 %max, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc3_umin_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_umin_i16_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 1
-; CHECK-NEXT: [[MIN:%.*]] = tail call i16 @llvm.smin.i16(i16 [[TMP1]], i16 [[TMP2]])
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[MIN:%.*]] = tail call i16 @llvm.umin.i16(i16 [[TMP0]], i16 [[TMP1]])
; CHECK-NEXT: store i16 [[MIN]], ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
@@ -483,7 +521,7 @@ loop:
%0 = load i16, ptr %arrayidx, align 1
%arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %iv
%1 = load i16, ptr %arrayidx2, align 1
- %min = tail call i16 @llvm.smin.i16(i16 %0, i16 %1)
+ %min = tail call i16 @llvm.umin.i16(i16 %0, i16 %1)
store i16 %min, ptr %arrayidx, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, 3
@@ -493,6 +531,118 @@ exit:
ret void
}
+define void @tc3_xor_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_xor_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: store i8 [[XOR]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %xor = xor i8 %0, %1
+ store i8 %xor, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @tc5_xor_i8_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc5_xor_i8_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[XOR:%.*]] = xor i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT: store i8 [[XOR]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %b, i64 %iv
+ %0 = load i8, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i8, ptr %a, i64 %iv
+ %1 = load i8, ptr %arrayidx2, align 1
+ %xor = xor i8 %0, %1
+ store i8 %xor, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 5
+ br i1 %exitcond, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @tc3_xor_i16_reject(ptr noalias %a, ptr noalias %b) #0 {
+; CHECK-LABEL: define void @tc3_xor_i16_reject(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[XOR:%.*]] = xor i16 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: store i16 [[XOR]], ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i16, ptr %b, i64 %iv
+ %0 = load i16, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i16, ptr %a, i64 %iv
+ %1 = load i16, ptr %arrayidx2, align 1
+ %xor = xor i16 %0, %1
+ store i16 %xor, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 3
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
More information about the llvm-commits
mailing list