[llvm] 4257da8 - [LV] Always try to use fixed VF for low TC loops if no epilogue allowed. (#226318)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 23:47:53 PDT 2026
Author: Florian Hahn
Date: 2026-09-26T06:47:43Z
New Revision: 4257da8ee2e84b3e62d2a2a5010e81823b799805
URL: https://github.com/llvm/llvm-project/commit/4257da8ee2e84b3e62d2a2a5010e81823b799805
DIFF: https://github.com/llvm/llvm-project/commit/4257da8ee2e84b3e62d2a2a5010e81823b799805.diff
LOG: [LV] Always try to use fixed VF for low TC loops if no epilogue allowed. (#226318)
Remove the MaxPowerOf2RuntimeVF gate for falling back to using fixed
width VFs when no epilogue is allowed and the trip count is below the
minimum for tail folding.
MaxPowerOf2RuntimeVF being not set means we cannot compute the maximum
runtime VF, due to missing max vscale. In that case we are not able to
determine if a epilogue loop remains for scalable VFs, but we can still
pick a fixed VF, if no epilogue remains for it.
Also updates processLoop to ignore CM_EpilogueNotNeededFoldTail, if the
trip count is below the tail-folding threshold.
PR: https://github.com/llvm/llvm-project/pull/226318
Added:
llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-fixed-vf-no-epilogue.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 81bffe11ea656..73588103e26d7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3056,17 +3056,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (ExpectedTC && ExpectedTC->isFixed() &&
ExpectedTC->getFixedValue() <=
TTI.getMinTripCountTailFoldingThreshold()) {
- if (MaxPowerOf2RuntimeVF > 0u) {
- // If we have a low-trip-count, and the fixed-width VF is known to divide
- // the trip count but the scalable factor does not, use the fixed-width
- // factor in preference to allow the generation of a non-predicated loop.
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
- NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
- LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
- "remain for any chosen VF.\n");
- MaxFactors.ScalableVF = ElementCount::getScalable(0);
- return MaxFactors;
- }
+ // If we have a low-trip-count, and the fixed-width VF is known to divide
+ // the trip count the fixed-width factor in preference to allow the
+ // generation of a non-predicated loop.
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
+ NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
+ LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
+ "remain for any chosen VF.\n");
+ MaxFactors.ScalableVF = ElementCount::getScalable(0);
+ return MaxFactors;
}
// Allow cases where the ExactTC == (VF * IC) or ExactTC == (VF * IC) + 1.
@@ -7813,8 +7811,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
// with runtime checks. It's more effective to let
// `isOutsideLoopWorkProfitable` determine if vectorization is
- // beneficial for the loop.
- if (SEL != CM_EpilogueNotNeededFoldTail)
+ // beneficial for the loop. If the trip count is below the target's
+ // minimum for tail-folding, the tail cannot be folded, so treat it like
+ // any other low trip count loop.
+ if (SEL != CM_EpilogueNotNeededFoldTail ||
+ ExpectedTC->getFixedValue() <=
+ TTI->getMinTripCountTailFoldingThreshold())
SEL = CM_EpilogueNotAllowedLowTripLoop;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-fixed-vf-no-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-fixed-vf-no-epilogue.ll
new file mode 100644
index 0000000000000..925fa17267249
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-fixed-vf-no-epilogue.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -tail-folding-policy=prefer-fold-tail -S %s | FileCheck %s
+
+; Trip count 4 is below the SVE tail-folding threshold. Pick a fixed-width VF
+; that divides the trip count.
+define void @tc4_no_vscale_range(ptr %a) {
+; CHECK-LABEL: define void @tc4_no_vscale_range(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %v = load i32, ptr %gep, align 4
+ %add = add nsw i32 %v, 1
+ store i32 %add, ptr %gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 4
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tc4_vscale_range(ptr %a) vscale_range(1,16) {
+; CHECK-LABEL: define void @tc4_vscale_range(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %v = load i32, ptr %gep, align 4
+ %add = add nsw i32 %v, 1
+ store i32 %add, ptr %gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 4
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Trip count 3 is not a multiple of any fixed-width VF.
+define void @tc3_no_vscale_range(ptr %a) {
+; CHECK-LABEL: define void @tc3_no_vscale_range(
+; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[V]], 1
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %v = load i32, ptr %gep, align 4
+ %add = add nsw i32 %v, 1
+ store i32 %add, ptr %gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 3
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
index b370fbe2da777..8a7fed496e6a3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
@@ -4,17 +4,17 @@
define void @small_trip_count_min_vlen_128(ptr nocapture %a) vscale_range(4,1024) {
; CHECK-LABEL: define void @small_trip_count_min_vlen_128(
; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
-; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[V]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV]], 3
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -38,17 +38,17 @@ exit:
define void @small_trip_count_min_vlen_32(ptr nocapture %a) vscale_range(1,1024) {
; CHECK-LABEL: define void @small_trip_count_min_vlen_32(
; CHECK-SAME: ptr captures(none) [[A:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
-; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[V]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[IV]], 3
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list