[llvm] [LV] Allow runtime checks for low trip count loops without scalar epilogues (PR #197391)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 03:22:11 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/197391
>From f8355d181a0b2c299fcd5e3b2ad1e8c3cf4ccd63 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 13 May 2026 01:20:40 -0700
Subject: [PATCH 1/6] Remove the hard limitation of low constant trip count for
runtime check
---
.../Transforms/Vectorize/LoopVectorize.cpp | 37 ++++++++-----------
.../AArch64/low_trip_count_predicates.ll | 4 +-
.../LoopVectorize/AArch64/pr36032.ll | 36 ++++++++++++++++--
.../runtime-check-trip-count-decisions.ll | 7 ++--
4 files changed, 54 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2402cc9491d5aa0..6484b720f7dd4bf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -191,12 +191,11 @@ static cl::opt<unsigned> EpilogueVectorizationMinVF(
"the specified value are considered for epilogue vectorization."));
/// Loops with a known constant trip count below this number are vectorized only
-/// if no scalar iteration overheads are incurred.
+/// if no scalar epilogue is required.
static cl::opt<unsigned> TinyTripCountVectorThreshold(
"vectorizer-min-trip-count", cl::init(16), cl::Hidden,
cl::desc("Loops with a constant trip count that is smaller than this "
- "value are vectorized only if no scalar iteration overheads "
- "are incurred."));
+ "value are vectorized only if no scalar epilogue is required"));
static cl::opt<bool> ForcePartialAliasingVectorization(
"force-partial-aliasing-vectorization", cl::init(false), cl::Hidden,
@@ -695,10 +694,9 @@ enum EpilogueLowering {
// Vectorization with OptForSize: don't allow epilogues.
CM_EpilogueNotAllowedOptSize,
- // A special case of vectorisation with OptForSize: loops with a very small
- // trip count are considered for vectorization under OptForSize, thereby
- // making sure the cost of their loop body is dominant, free of runtime
- // guards and scalar iteration overheads.
+ // A special case for loops with a very small trip count: they are vectorized
+ // only if no scalar epilogue is required, ensuring the vectorized loop body
+ // is the dominant cost without scalar epilogue overheads.
CM_EpilogueNotAllowedLowTripLoop,
// Loop hint indicating an epilogue is undesired, apply tail folding.
@@ -2935,13 +2933,11 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
<< "vector loop.\n");
break;
case CM_EpilogueNotAllowedLowTripLoop:
- // fallthrough as a special case of OptForSize
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
+ << "count.\n");
+ break;
case CM_EpilogueNotAllowedOptSize:
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
- LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
- else
- LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
- << "count.\n");
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
// Bail if runtime checks are required, which are not good when optimising
// for size.
@@ -7753,19 +7749,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is worth vectorizing only if no scalar "
- << "iteration overheads are incurred.");
+ << "epilogue is required.");
if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
LLVM_DEBUG(dbgs() << "\n");
- // Tail-folded loops are efficient even when the loop
- // iteration count is low. However, setting the epilogue policy to
- // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
- // with runtime checks. It's more effective to let
- // `isOutsideLoopWorkProfitable` determine if vectorization is
- // beneficial for the loop. If the trip count is below the target's
- // minimum for tail-folding, the tail cannot be folded, so treat it like
- // any other low trip count loop.
+ // For loops with very small trip counts, require that no scalar epilogue
+ // is needed. Tail-folded loops are still allowed since they avoid
+ // epilogues, unless the trip count is below the target's minimum for
+ // tail-folding. In that case the tail cannot be folded, so the loop
+ // should be treated like any other low trip count loop.
if (SEL != CM_EpilogueNotNeededFoldTail ||
ExpectedTC->getFixedValue() <=
TTI->getMinTripCountTailFoldingThreshold())
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index 2f5e61cb1d2360e..525f8ee99b42cd5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -17,8 +17,8 @@ target triple = "aarch64-unknown-linux-gnu"
; DEBUG-VS2: Main Loop VF:vscale x 8, Main Loop UF:1, Epilogue Loop VF:8, Epilogue Loop UF:1
; DEBUG-LABEL: LV: Checking a loop in 'trip_count_too_small'
-; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar iteration overheads are incurred.
-; DEBUG: LV: Not vectorizing: Runtime SCEV check is required with -Os/-Oz.
+; DEBUG: LV: Found a loop with a very small trip count. This loop is worth vectorizing only if no scalar epilogue is required.
+; DEBUG: LV: Not vectorizing: The trip count is below the minial threshold value..
; DEBUG-LABEL: LV: Checking a loop in 'too_many_runtime_checks'
; DEBUG: LV: Found trip count: 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
index f4a1814efc03842..56c017374b4651f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
@@ -14,6 +14,7 @@ define void @_Z1dv() {
; CHECK-LABEL: @_Z1dv(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CALL:%.*]] = tail call ptr @"_ZN3$_01aEv"(ptr nonnull @b)
+; CHECK-NEXT: [[CALL1:%.*]] = ptrtoaddr ptr [[CALL]] to i64
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: for.cond:
; CHECK-NEXT: [[F_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD5:%.*]], [[FOR_COND_CLEANUP:%.*]] ]
@@ -23,15 +24,44 @@ define void @_Z1dv() {
; CHECK-NEXT: br i1 [[CMP12]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP]]
; CHECK: for.body.lr.ph:
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[G_0]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = sub nsw i64 4, [[TMP0]]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[CALL1]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[G_0]], [[CONV]]
+; CHECK-NEXT: [[TMP4:%.*]] = zext nneg i32 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], ptrtoaddr (ptr @c to i64)
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], 3
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP12]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP12]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[CONV]], [[G_0]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i8], ptr @c, i64 0, i64 [[TMP9]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[CALL]], i64 [[TMP0]]
+; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD]], ptr [[TMP11]], align 1
+; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[FOR_BODY]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
; CHECK: for.cond.cleanup.loopexit:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[G_1_LCSSA]] = phi i32 [ [[G_0]], [[FOR_COND]] ], [ 4, [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT: [[G_1_LCSSA]] = phi i32 [ [[G_0]], [[FOR_COND]] ], [ 4, [[FOR_COND_CLEANUP_LOOPEXIT]] ]
; CHECK-NEXT: [[ADD5]] = add nuw nsw i32 [[CONV]], 4
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK: for.body:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[CONV]], [[TMP1]]
; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[ADD]] to i64
@@ -41,7 +71,7 @@ define void @_Z1dv() {
; CHECK-NEXT: store i8 [[TMP2]], ptr [[ARRAYIDX3]], align 1
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 4
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
;
entry:
%call = tail call ptr @"_ZN3$_01aEv"(ptr nonnull @b) #2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
index a1ced9e323fee57..60cb1410222f729 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
@@ -79,12 +79,13 @@ for.end:
ret i32 0
}
-; If trip-count is equal to 12, the function is vectorised when predicated tail
-; folding is chosen.
+; If trip-count is equal to 12, the function is vectorised, as the cost
+; model decides whether the required runtime check is worthwhile, instead
+; of unconditionally rejecting low trip-count loops that need one.
define i32 @foo_mid_trip_count_12(ptr %a, ptr %b, ptr %c, i32 %bound) {
; CHECK-LABEL: @foo_mid_trip_count_12(
; PREDICATED: vector.body
-; SCALAR-NOT: vector.body
+; SCALAR: vector.body
entry:
br label %for.body
>From 20faa84990a5b39265acf35a12297ae48065df71 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 21 Jul 2026 01:12:29 -0700
Subject: [PATCH 2/6] Add test case with non const trip count
this case has non const trip count, but pgo profile shows the trip count
could be 12 (lower than 16). Cost model will choose VF=4, and emit min
profitable TC check.
---
.../AArch64/low-trip-count-runtime-check.ll | 87 +++++++++++++++++++
1 file changed, 87 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
new file mode 100644
index 000000000000000..3ef4086dffdc201
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low-trip-count-runtime-check.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu | FileCheck %s
+
+define void @low_trip_count_12_needs_runtime_check(ptr %dst, ptr %a, ptr %b, i32 %m) {
+; CHECK-LABEL: define void @low_trip_count_12_needs_runtime_check(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B3:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[N:%.*]] = mul nuw i32 [[M]], 4
+; CHECK-NEXT: [[GUARD:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[GUARD]], label %[[EXIT:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[A2]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[DST1]], [[B3]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT: [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP3]], 15
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[LOOP_PH]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
+; CHECK-NEXT: [[AV:%.*]] = load i32, ptr [[A_GEP]], align 4
+; CHECK-NEXT: [[B_GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[IV]]
+; CHECK-NEXT: [[BV:%.*]] = load i32, ptr [[B_GEP]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[AV]], [[BV]]
+; CHECK-NEXT: [[D_GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: store i32 [[SUM]], ptr [[D_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %n = mul nuw i32 %m, 4
+ %guard = icmp eq i32 %n, 0
+ br i1 %guard, label %exit, label %loop.ph
+
+loop.ph:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %loop.ph ], [ %iv.next, %loop ]
+ %a.gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %av = load i32, ptr %a.gep, align 4
+ %b.gep = getelementptr inbounds i32, ptr %b, i32 %iv
+ %bv = load i32, ptr %b.gep, align 4
+ %sum = add i32 %av, %bv
+ %d.gep = getelementptr inbounds i32, ptr %dst, i32 %iv
+ store i32 %sum, ptr %d.gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 110, i32 10}
>From cfda0d4beefa582cad9a7ff8e3b96f12b8c5ac4c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 11 Aug 2026 23:49:29 -0700
Subject: [PATCH 3/6] rebase and update
---
llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
index 56c017374b4651f..59d1658bb20914e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
@@ -36,7 +36,7 @@ define void @_Z1dv() {
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], 3
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP12]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP12]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP12]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
>From c2121318ecba7fd103ddd260547b94d748d6b3c3 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Wed, 12 Aug 2026 01:21:32 -0700
Subject: [PATCH 4/6] fix optForSize
---
.../Transforms/Vectorize/LoopVectorize.cpp | 16 ++-
.../LoopVectorize/AArch64/optsize_minsize.ll | 99 +++++++++++++++++++
2 files changed, 112 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6484b720f7dd4bf..2a924edec7bf2d4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7759,9 +7759,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// epilogues, unless the trip count is below the target's minimum for
// tail-folding. In that case the tail cannot be folded, so the loop
// should be treated like any other low trip count loop.
- if (SEL != CM_EpilogueNotNeededFoldTail ||
- ExpectedTC->getFixedValue() <=
- TTI->getMinTripCountTailFoldingThreshold())
+ // TODO: Remove the check for CM_EpilogueNotAllowedOptSize here. The
+ // EpilogueLowering enum currently conflates several pieces of
+ // information:
+ // * Whether a scalar epilogue is permitted
+ // * Whether tail folding is preferred or required
+ // * Whether optimizing for code size
+ // Overwriting it here can drop OptForSize, breaking the
+ // runtimeChecksRequired() bail-out in computeMaxVF. Split
+ // EpilogueLowering into separate fields instead.
+ if (SEL != CM_EpilogueNotAllowedOptSize &&
+ (SEL != CM_EpilogueNotNeededFoldTail ||
+ ExpectedTC->getFixedValue() <=
+ TTI->getMinTripCountTailFoldingThreshold()))
SEL = CM_EpilogueNotAllowedLowTripLoop;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index cb79aff9fed80a1..e19c02f9d122e73 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -776,6 +776,105 @@ for.cond.cleanup:
ret void
}
+; A low constant trip count loop that needs memory runtime checks.
+define void @low_trip_count_needs_runtime_checks(ptr %p, ptr %q) {
+; DEFAULT-LABEL: define void @low_trip_count_needs_runtime_checks(
+; DEFAULT-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; DEFAULT: [[VECTOR_MEMCHECK]]:
+; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 48
+; DEFAULT-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[Q]], i64 48
+; DEFAULT-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP1]]
+; DEFAULT-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP]]
+; DEFAULT-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; DEFAULT-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META7:![0-9]+]], !noalias [[META10:![0-9]+]]
+; DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META10]]
+; DEFAULT-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; DEFAULT-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4, !alias.scope [[META7]], !noalias [[META10]]
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; DEFAULT-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
+; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT: [[SCALAR_PH]]:
+; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
+; DEFAULT: [[FOR_BODY]]:
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IV]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[IV]]
+; DEFAULT-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; DEFAULT-NEXT: [[ADD:%.*]] = add i32 [[TMP4]], [[TMP5]]
+; DEFAULT-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX]], align 4
+; DEFAULT-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 12
+; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; DEFAULT: [[FOR_COND_CLEANUP]]:
+; DEFAULT-NEXT: ret void
+;
+; OPTSIZE-LABEL: define void @low_trip_count_needs_runtime_checks(
+; OPTSIZE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; OPTSIZE-NEXT: [[ENTRY:.*]]:
+; OPTSIZE-NEXT: br label %[[FOR_BODY:.*]]
+; OPTSIZE: [[FOR_BODY]]:
+; OPTSIZE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IV]]
+; OPTSIZE-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; OPTSIZE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[IV]]
+; OPTSIZE-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; OPTSIZE-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], [[TMP1]]
+; OPTSIZE-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX]], align 4
+; OPTSIZE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; OPTSIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 12
+; OPTSIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]]
+; OPTSIZE: [[FOR_COND_CLEANUP]]:
+; OPTSIZE-NEXT: ret void
+;
+; MINSIZE-LABEL: define void @low_trip_count_needs_runtime_checks(
+; MINSIZE-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; MINSIZE-NEXT: [[ENTRY:.*]]:
+; MINSIZE-NEXT: br label %[[FOR_BODY:.*]]
+; MINSIZE: [[FOR_BODY]]:
+; MINSIZE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IV]]
+; MINSIZE-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; MINSIZE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[IV]]
+; MINSIZE-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; MINSIZE-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], [[TMP1]]
+; MINSIZE-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX]], align 4
+; MINSIZE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; MINSIZE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 12
+; MINSIZE-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]]
+; MINSIZE: [[FOR_COND_CLEANUP]]:
+; MINSIZE-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds i32, ptr %q, i64 %iv
+ %1 = load i32, ptr %arrayidx2, align 4
+ %add = add i32 %0, %1
+ store i32 %add, ptr %arrayidx, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 12
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 84371320bf30571620301127ab6022ebb2b3b0d0 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 14 Sep 2026 02:02:24 -0700
Subject: [PATCH 5/6] rebase and update
---
.../AArch64/non-affine-monotonic-bounds.ll | 25 ++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/non-affine-monotonic-bounds.ll b/llvm/test/Transforms/LoopVectorize/AArch64/non-affine-monotonic-bounds.ll
index 7ccd5da926c219a..e85cd7aab8f6f66 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/non-affine-monotonic-bounds.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/non-affine-monotonic-bounds.ll
@@ -44,10 +44,29 @@ exit:
define void @bitset_tc8(ptr %words, ptr %out) {
; CHECK-LABEL: define void @bitset_tc8(
; CHECK-SAME: ptr [[WORDS:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 8
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[WORDS]], i64 1
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[WORDS]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[WORDS]], align 1, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: store <8 x i8> [[BROADCAST_SPLAT]], ptr [[OUT]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[DIV:%.*]] = lshr i64 [[IV]], 6
; CHECK-NEXT: [[GEP_WORDS:%.*]] = getelementptr inbounds i8, ptr [[WORDS]], i64 [[DIV]]
; CHECK-NEXT: [[W:%.*]] = load i8, ptr [[GEP_WORDS]], align 1
@@ -55,7 +74,7 @@ define void @bitset_tc8(ptr %words, ptr %out) {
; CHECK-NEXT: store i8 [[W]], ptr [[GEP_OUT]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 8
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From c6ef54a93557a32aad323fe85d475660007835da Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 14 Sep 2026 02:03:23 -0700
Subject: [PATCH 6/6] remove CM_EpilogueNotAllowedLowTripLoop from
finalizeSCEVPredicates
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/LoopVectorize/X86/optsize.ll | 88 ++++++++++++++-----
.../pr39417-optsize-scevchecks.ll | 39 +++++---
3 files changed, 97 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2a924edec7bf2d4..db82501bf20fcea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6391,10 +6391,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// Add surviving induction predicates to PSE and check constraints.
bool ForceVectorization =
Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
- bool OptForSize =
- !ForceVectorization &&
- (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
- CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
+ bool OptForSize = !ForceVectorization &&
+ CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize;
unsigned SCEVCheckThreshold = ForceVectorization
? PragmaVectorizeSCEVCheckThreshold
: VectorizeSCEVCheckThreshold;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index c78688c348b8400..bc90a963009fc9d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -230,16 +230,62 @@ attributes #2 = { optsize }
define void @scev_predicate_no_vec(i32 %start, ptr %dst) {
; CHECK-LABEL: define void @scev_predicate_no_vec(
; CHECK-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[START]] to i16
+; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 4)
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], [[TMP1]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[START]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
+; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP6]], i32 4)
+; CHECK-NEXT: [[TMP7:%.*]] = sub i32 [[UMAX]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i16
+; CHECK-NEXT: [[TMP9:%.*]] = add i16 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ult i16 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt i32 [[TMP7]], 65535
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], [[TMP6]]
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP12]], [[IDENT_CHECK]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP4]], 63
+; CHECK-NEXT: [[TMP14:%.*]] = and i32 [[N_RND_UP]], 63
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[TMP14]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <64 x i32> poison, i32 [[START]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT1]], <64 x i32> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <64 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <64 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <64 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ule <64 x i32> [[VEC_IND3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[IV:%.*]] = add i32 [[START]], [[INDEX]]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[IV]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[CONV:%.*]] = and i32 [[IV]], 65535
+; CHECK-NEXT: call void @llvm.masked.store.v64i32.p0(<64 x i32> [[VEC_IND]], ptr align 4 [[GEP]], <64 x i1> [[TMP15]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <64 x i32> [[VEC_IND]], splat (i32 64)
+; CHECK-NEXT: [[VEC_IND_NEXT4]] = add nuw <64 x i32> [[VEC_IND3]], splat (i32 64)
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ [[START]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV1]]
+; CHECK-NEXT: store i32 [[IV1]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[CONV:%.*]] = and i32 [[IV1]], 65535
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], 4
; CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[CONV]], 1
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -277,32 +323,30 @@ exit:
define void @can_prove_scev_predicate_is_always_true(ptr %dst) {
; CHECK-LABEL: define void @can_prove_scev_predicate_is_always_true(
; CHECK-SAME: ptr [[DST:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[IV]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[CONV:%.*]] = and i32 [[IV]], 65535
-; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], 4
-; CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[CONV]], 1
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: call void @llvm.masked.store.v64i32.p0(<64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>, ptr align 4 [[DST]], <64 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
; AUTOVF-LABEL: define void @can_prove_scev_predicate_is_always_true(
; AUTOVF-SAME: ptr [[DST:%.*]]) #[[ATTR2]] {
-; AUTOVF-NEXT: [[ENTRY:.*]]:
+; AUTOVF-NEXT: [[ENTRY:.*:]]
; AUTOVF-NEXT: br label %[[LOOP:.*]]
; AUTOVF: [[LOOP]]:
-; AUTOVF-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
-; AUTOVF-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; AUTOVF-NEXT: store i32 [[IV]], ptr [[GEP]], align 4
-; AUTOVF-NEXT: [[CONV:%.*]] = and i32 [[IV]], 65535
-; AUTOVF-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], 4
-; AUTOVF-NEXT: [[ADD]] = add nuw nsw i32 [[CONV]], 1
-; AUTOVF-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; AUTOVF-NEXT: br label %[[EXIT:.*]]
; AUTOVF: [[EXIT]]:
+; AUTOVF-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, ptr align 4 [[DST]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; AUTOVF-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; AUTOVF: [[MIDDLE_BLOCK]]:
+; AUTOVF-NEXT: br label %[[EXIT1:.*]]
+; AUTOVF: [[EXIT1]]:
; AUTOVF-NEXT: ret void
;
entry:
@@ -351,7 +395,7 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -4608
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i32> [[VEC_IV]], splat (i32 64)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
index 2f7024d2af63d86..3c6ee519f0c8d5e 100644
--- a/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
@@ -9,19 +9,38 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define void @func_34(ptr %dst) {
; CHECK-LABEL: define void @func_34(
; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SEXT:%.*]] = shl i32 [[IV]], 16
-; CHECK-NEXT: [[STEP:%.*]] = ashr exact i32 [[SEXT]], 16
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[STEP]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[STEP]], 1
-; CHECK-NEXT: [[IV_NEXT_TRUNC:%.*]] = trunc i32 [[IV_NEXT]] to i16
-; CHECK-NEXT: [[EC:%.*]] = icmp slt i16 [[IV_NEXT_TRUNC]], 3
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 1
+; CHECK-NEXT: store i32 1, ptr [[TMP0]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 2
+; CHECK-NEXT: store i32 2, ptr [[TMP1]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 3
+; CHECK-NEXT: store i32 3, ptr [[TMP2]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list