[llvm] [LV] Avoid low-trip-count vectorization for countable early exits (PR #225635)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 01:19:25 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/225635
>From 8e1e10b8d548b761e4e58574d598389e448d9732 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 23 Sep 2026 08:10:42 +0100
Subject: [PATCH 1/2] [LV] Avoid low-trip-count vectorization for countable
early exits
Following #195823, If a loop has a TC that is equal to (VF * IC) + 1,
we are able to vectorize the loop to produce 1 scalar and 1 vector
iteration. However, for loops that have an early exit condition, this
can lead to miscompiles as it will run a full vector iteration, rather
than just the iterations intended.
For loops where an early exit condition is present, and the exiting
block of the loop is not in the same block as the loop latch, the
new transformation should not be applied.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
.../AArch64/sve-low-trip-count-early-exit.ll | 50 +++++++++++++++++++
.../RISCV/low-trip-count-early-exit.ll | 50 +++++++++++++++++++
3 files changed, 102 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count-early-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count-early-exit.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 805a57f8dc4ab..eff6fe1fe7c67 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3090,7 +3090,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
unsigned MaxVFForTC = llvm::bit_floor(TC.getFixedValue());
if (TC.getFixedValue() - MaxVFForTC <= 1 && MaxVFForTC / EffectiveIC > 1 &&
MaxVFForTC <= (MaxFactors.FixedVF.getFixedValue() * EffectiveIC) &&
- !Config.OptForSize) {
+ !Config.OptForSize &&
+ TheLoop->getExitingBlock() == TheLoop->getLoopLatch()) {
unsigned NumOfInstructions = llvm::sum_of(
llvm::map_range(TheLoop->blocks(),
[](BasicBlock *BB) { return BB->size(); }),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count-early-exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count-early-exit.ll
new file mode 100644
index 0000000000000..dbb1f62098fe8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count-early-exit.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -low-trip-count-loop-body-size-limit=0 < %s | FileCheck %s
+
+define i32 @trip5_early_exit(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) {
+; CHECK-LABEL: define i32 @trip5_early_exit(
+; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[A:%.*]] = icmp eq i64 [[IV]], 3
+; CHECK-NEXT: br i1 [[A]], label %[[EXIT1:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
+; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT2:.*]], label %[[LOOP]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret i32 1
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: ret i32 2
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %a = icmp eq i64 %iv, 3
+ br i1 %a, label %exit1, label %latch
+latch:
+ %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
+ %0 = load i8, ptr %gep.src, align 1
+ %mul = shl i8 %0, 1
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ %1 = load i8, ptr %gep.dst, align 1
+ %add = add i8 %mul, %1
+ store i8 %add, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 5
+ br i1 %ec, label %exit2, label %loop
+exit1:
+ ret i32 1
+exit2:
+ ret i32 2
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count-early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count-early-exit.ll
new file mode 100644
index 0000000000000..1168aaed0612f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count-early-exit.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -low-trip-count-loop-body-size-limit=0 < %s | FileCheck %s
+
+define i32 @trip5_early_exit(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) {
+; CHECK-LABEL: define i32 @trip5_early_exit(
+; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[A:%.*]] = icmp eq i64 [[IV]], 3
+; CHECK-NEXT: br i1 [[A]], label %[[EXIT1:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT: [[MUL:%.*]] = shl i8 [[TMP0]], 1
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
+; CHECK-NEXT: store i8 [[ADD]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT2:.*]], label %[[LOOP]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret i32 1
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: ret i32 2
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %a = icmp eq i64 %iv, 3
+ br i1 %a, label %exit1, label %latch
+latch:
+ %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
+ %0 = load i8, ptr %gep.src, align 1
+ %mul = shl i8 %0, 1
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ %1 = load i8, ptr %gep.dst, align 1
+ %add = add i8 %mul, %1
+ store i8 %add, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 5
+ br i1 %ec, label %exit2, label %loop
+exit1:
+ ret i32 1
+exit2:
+ ret i32 2
+}
>From 0956e6eb0f1350a6c323016571612c39777dcf78 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 23 Sep 2026 09:16:16 +0100
Subject: [PATCH 2/2] Update new test
---
.../RISCV/countable-early-exit-no-epilogue.ll | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/countable-early-exit-no-epilogue.ll b/llvm/test/Transforms/LoopVectorize/RISCV/countable-early-exit-no-epilogue.ll
index 349a026cc1b4a..7b6f335af13c0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/countable-early-exit-no-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/countable-early-exit-no-epilogue.ll
@@ -7,16 +7,19 @@
define i32 @countable_early_exit(ptr noalias %b) {
; NO-EPILOGUE-LABEL: define i32 @countable_early_exit(
; NO-EPILOGUE-SAME: ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; NO-EPILOGUE-NEXT: [[ENTRY:.*:]]
-; NO-EPILOGUE-NEXT: br label %[[VECTOR_PH:.*]]
-; NO-EPILOGUE: [[VECTOR_PH]]:
+; NO-EPILOGUE-NEXT: [[VECTOR_PH:.*]]:
; NO-EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-EPILOGUE: [[VECTOR_BODY]]:
-; NO-EPILOGUE-NEXT: store <4 x i32> splat (i32 1), ptr [[B]], align 4
-; NO-EPILOGUE-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; NO-EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; NO-EPILOGUE-NEXT: [[C:%.*]] = icmp eq i64 [[IV]], 3
+; NO-EPILOGUE-NEXT: br i1 [[C]], label %[[EXIT1:.*]], label %[[MIDDLE_BLOCK]]
; NO-EPILOGUE: [[MIDDLE_BLOCK]]:
-; NO-EPILOGUE-NEXT: br label %[[EXIT2:.*]]
-; NO-EPILOGUE: [[EXIT1:.*:]]
+; NO-EPILOGUE-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-EPILOGUE-NEXT: store i32 1, ptr [[GEP]], align 4
+; NO-EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; NO-EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT2:.*]], label %[[VECTOR_BODY]]
+; NO-EPILOGUE: [[EXIT1]]:
; NO-EPILOGUE-NEXT: ret i32 1
; NO-EPILOGUE: [[EXIT2]]:
; NO-EPILOGUE-NEXT: ret i32 2
More information about the llvm-commits
mailing list