[llvm] [LV] Allow runtime checks for low trip count loops without scalar epilogues (PR #197391)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 21 01:20:56 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/197391
>From 5f9895379f1ebe71dfc62620210d24492a0d0a4e Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 13 May 2026 01:20:40 -0700
Subject: [PATCH 1/2] Remove the hard limitation of low constant trip count for
runtime check
---
.../Transforms/Vectorize/LoopVectorize.cpp | 32 +++++++----------
.../AArch64/low_trip_count_predicates.ll | 4 +--
.../LoopVectorize/AArch64/pr36032.ll | 36 +++++++++++++++++--
.../runtime-check-trip-count-decisions.ll | 7 ++--
4 files changed, 51 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4cd6a55f01a90..6f1e23823a6f9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -191,12 +191,11 @@ static cl::opt<unsigned> EpilogueVectorizationMinVF(
"the specified value are considered for epilogue vectorization."));
/// Loops with a known constant trip count below this number are vectorized only
-/// if no scalar iteration overheads are incurred.
+/// if no scalar epilogue is required.
static cl::opt<unsigned> TinyTripCountVectorThreshold(
"vectorizer-min-trip-count", cl::init(16), cl::Hidden,
cl::desc("Loops with a constant trip count that is smaller than this "
- "value are vectorized only if no scalar iteration overheads "
- "are incurred."));
+ "value are vectorized only if no scalar epilogue is required"));
static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
@@ -758,10 +757,9 @@ enum EpilogueLowering {
// Vectorization with OptForSize: don't allow epilogues.
CM_EpilogueNotAllowedOptSize,
- // A special case of vectorisation with OptForSize: loops with a very small
- // trip count are considered for vectorization under OptForSize, thereby
- // making sure the cost of their loop body is dominant, free of runtime
- // guards and scalar iteration overheads.
+ // A special case for loops with a very small trip count: they are vectorized
+ // only if no scalar epilogue is required, ensuring the vectorized loop body
+ // is the dominant cost without scalar epilogue overheads.
CM_EpilogueNotAllowedLowTripLoop,
// Loop hint indicating an epilogue is undesired, apply tail folding.
@@ -2992,13 +2990,11 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
<< "vector loop.\n");
break;
case CM_EpilogueNotAllowedLowTripLoop:
- // fallthrough as a special case of OptForSize
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
+ << "count.\n");
+ break;
case CM_EpilogueNotAllowedOptSize:
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
- LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
- else
- LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
- << "count.\n");
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
// Bail if runtime checks are required, which are not good when optimising
// for size.
@@ -7970,17 +7966,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is worth vectorizing only if no scalar "
- << "iteration overheads are incurred.");
+ << "epilogue is required.");
if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
LLVM_DEBUG(dbgs() << "\n");
- // Tail-folded loops are efficient even when the loop
- // iteration count is low. However, setting the epilogue policy to
- // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
- // with runtime checks. It's more effective to let
- // `isOutsideLoopWorkProfitable` determine if vectorization is
- // beneficial for the loop.
+ // For loops with very small trip counts, require that no scalar epilogue
+ // is needed. Tail-folded loops are still allowed as they avoid epilogues.
if (SEL != CM_EpilogueNotNeededFoldTail)
SEL = CM_EpilogueNotAllowedLowTripLoop;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index f123ed20e205c..6ca7ed92275a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -17,8 +17,8 @@ target triple = "aarch64-unknown-linux-gnu"
; DEBUG-VS2: Main Loop VF:vscale x 8, Main Loop UF:1, Epilogue Loop VF:8, Epilogue Loop UF:1
; DEBUG-LABEL: LV: Checking a loop in 'trip_count_too_small'
-; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar iteration overheads are incurred.
-; DEBUG: LV: Not vectorizing: Runtime SCEV check is required with -Os/-Oz.
+; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar epilogue is required.
+; DEBUG: LV: Not vectorizing: The trip count is below the minial threshold value..
; DEBUG-LABEL: LV: Checking a loop in 'too_many_runtime_checks'
; DEBUG: LV: Found trip count: 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
index f4a1814efc038..56c017374b465 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
@@ -14,6 +14,7 @@ define void @_Z1dv() {
; CHECK-LABEL: @_Z1dv(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CALL:%.*]] = tail call ptr @"_ZN3$_01aEv"(ptr nonnull @b)
+; CHECK-NEXT: [[CALL1:%.*]] = ptrtoaddr ptr [[CALL]] to i64
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: for.cond:
; CHECK-NEXT: [[F_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD5:%.*]], [[FOR_COND_CLEANUP:%.*]] ]
@@ -23,15 +24,44 @@ define void @_Z1dv() {
; CHECK-NEXT: br i1 [[CMP12]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP]]
; CHECK: for.body.lr.ph:
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[G_0]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = sub nsw i64 4, [[TMP0]]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[CALL1]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[G_0]], [[CONV]]
+; CHECK-NEXT: [[TMP4:%.*]] = zext nneg i32 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], ptrtoaddr (ptr @c to i64)
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], 3
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP12]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP12]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[CONV]], [[G_0]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i8], ptr @c, i64 0, i64 [[TMP9]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 [[TMP0]]
+; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD]], ptr [[TMP11]], align 1
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[FOR_BODY]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
; CHECK: for.cond.cleanup.loopexit:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[G_1_LCSSA]] = phi i32 [ [[G_0]], [[FOR_COND]] ], [ 4, [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT: [[G_1_LCSSA]] = phi i32 [ [[G_0]], [[FOR_COND]] ], [ 4, [[FOR_COND_CLEANUP_LOOPEXIT]] ]
; CHECK-NEXT: [[ADD5]] = add nuw nsw i32 [[CONV]], 4
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[CONV]], [[TMP1]]
; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[ADD]] to i64
@@ -41,7 +71,7 @@ define void @_Z1dv() {
; CHECK-NEXT: store i8 [[TMP2]], ptr [[ARRAYIDX3]], align 1
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 4
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
;
entry:
%call = tail call ptr @"_ZN3$_01aEv"(ptr nonnull @b) #2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
index a1ced9e323fee..60cb1410222f7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
@@ -79,12 +79,13 @@ for.end:
ret i32 0
}
-; If trip-count is equal to 12, the function is vectorised when predicated tail
-; folding is chosen.
+; If trip-count is equal to 12, the function is vectorised, as the cost
+; model decides whether the required runtime check is worthwhile, instead
+; of unconditionally rejecting low trip-count loops that need one.
define i32 @foo_mid_trip_count_12(ptr %a, ptr %b, ptr %c, i32 %bound) {
; CHECK-LABEL: @foo_mid_trip_count_12(
; PREDICATED: vector.body
-; SCALAR-NOT: vector.body
+; SCALAR: vector.body
entry:
br label %for.body
>From 644c761513a00a9b8af07306f59148b2297de33b Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 21 Jul 2026 01:12:29 -0700
Subject: [PATCH 2/2] Add test case with non const trip count
this case has non const trip count, but pgo profile shows the trip count
could be 12 (lower than 16). Cost model will choose VF=4, and emit min
profitable TC check.
---
.../AArch64/low-trip-count-runtime-check.ll | 87 +++++++++++++++++++
1 file changed, 87 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
new file mode 100644
index 0000000000000..3ef4086dffdc2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu | FileCheck %s
+
+define void @low_trip_count_12_needs_runtime_check(ptr %dst, ptr %a, ptr %b, i32 %m) {
+; CHECK-LABEL: define void @low_trip_count_12_needs_runtime_check(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B3:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[N:%.*]] = mul nuw i32 [[M]], 4
+; CHECK-NEXT: [[GUARD:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[GUARD]], label %[[EXIT:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[A2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[DST1]], [[B3]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP3]], 15
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[LOOP_PH]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
+; CHECK-NEXT: [[AV:%.*]] = load i32, ptr [[A_GEP]], align 4
+; CHECK-NEXT: [[B_GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[IV]]
+; CHECK-NEXT: [[BV:%.*]] = load i32, ptr [[B_GEP]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[AV]], [[BV]]
+; CHECK-NEXT: [[D_GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: store i32 [[SUM]], ptr [[D_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %n = mul nuw i32 %m, 4
+ %guard = icmp eq i32 %n, 0
+ br i1 %guard, label %exit, label %loop.ph
+
+loop.ph:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %loop.ph ], [ %iv.next, %loop ]
+ %a.gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %av = load i32, ptr %a.gep, align 4
+ %b.gep = getelementptr inbounds i32, ptr %b, i32 %iv
+ %bv = load i32, ptr %b.gep, align 4
+ %sum = add i32 %av, %bv
+ %d.gep = getelementptr inbounds i32, ptr %dst, i32 %iv
+ store i32 %sum, ptr %d.gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 110, i32 10}
More information about the llvm-commits
mailing list