[llvm] [AArch64] Cap upper-bound unrolling of loops with uncomputable trip counts (PR #205102)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 02:41:10 PDT 2026
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/205102
>From e6f5e96f5f68c74882d3d891ad6f1632642fdb77 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 19 Jun 2026 11:31:27 +0000
Subject: [PATCH 1/5] [LoopUnroll][AArch64] Add separate upper-bound limit for
loops with early exits
UnrollingPreferences::MaxUpperBound does not distinguish loops with early
exits from loops that run exactly 0 or N times (N <= MaxUpperBound).
Bounded (upper-bound) unrolling keeps all but the last loop test, so for
loops that may exit on any iteration it turns a single in-loop branch into
N branches. When such loops usually exit early, unrolling replaces a single
well-predicted branch with several rarely-taken ones, adding
branch-predictor and code-size pressure for little benefit.
Introduce UnrollingPreferences::MaxUpperBoundWithEarlyExits and the
-unroll-max-upperbound-with-early-exits option to bound such loops
separately. It defaults to MaxUpperBound, so behavior is unchanged unless a
target or the user lowers it. Loops known to run either the max count or
zero times (MaxOrZero) keep using MaxUpperBound.
Set the AArch64 limit to 5 to avoid bloating small early-exit loops such as
loops that count the number of bytes in a variable-length integer encoding.
---
.../llvm/Analysis/TargetTransformInfo.h | 3 +
.../AArch64/AArch64TargetTransformInfo.cpp | 1 +
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 16 +-
.../unroll-max-upperbound-with-early-exits.ll | 142 ++++++++++++++++++
.../unroll-max-upperbound-with-early-exits.ll | 137 +++++++++++++++++
5 files changed, 297 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..c18b246f0d633 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -697,6 +697,9 @@ class TargetTransformInfo {
/// to be overrided by a target gives more flexiblity on certain cases.
/// By default, MaxUpperBound uses UnrollMaxUpperBound which value is 8.
unsigned MaxUpperBound;
+ /// Set the maximum upper bound of trip count for loops that may exit before
+ /// reaching the bound. This defaults to MaxUpperBound.
+ unsigned MaxUpperBoundWithEarlyExits;
/// Set the maximum unrolling factor for full unrolling. Like MaxCount, but
/// applies even if full unrolling is selected. This allows a target to fall
/// back to Partial unrolling if full unrolling is above FullUnrollMaxCount.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ead9853f1da47..aa6e95d0ab816 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6041,6 +6041,7 @@ void AArch64TTIImpl::getUnrollingPreferences(
BaseT::getUnrollingPreferences(L, SE, UP, ORE);
UP.UpperBound = true;
+ UP.MaxUpperBoundWithEarlyExits = 5;
// For inner loop, it is more likely to be a hot one, and the runtime check
// can be promoted out from LICM pass, so the overhead is less, let's try
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index ad3b123f3327c..21374a6476db7 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -141,6 +141,11 @@ static cl::opt<unsigned> UnrollMaxUpperBound(
cl::desc(
"The max of trip count upper bound that is considered in unrolling"));
+static cl::opt<unsigned> UnrollMaxUpperBoundWithEarlyExits(
+ "unroll-max-upperbound-with-early-exits", cl::Hidden,
+ cl::desc("The max of trip count upper bound that is considered in "
+ "unrolling loops that may exit before reaching the bound"));
+
static cl::opt<unsigned> PragmaUnrollThreshold(
"pragma-unroll-threshold", cl::init(16 * 1024), cl::Hidden,
cl::desc("Unrolled size limit for loops with unroll metadata "
@@ -205,6 +210,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.DefaultUnrollRuntimeCount = 8;
UP.MaxCount = std::numeric_limits<unsigned>::max();
UP.MaxUpperBound = UnrollMaxUpperBound;
+ UP.MaxUpperBoundWithEarlyExits = UP.MaxUpperBound;
UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
UP.BEInsns = 2;
UP.Partial = false;
@@ -245,8 +251,12 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;
if (UnrollMaxCount.getNumOccurrences() > 0)
UP.MaxCount = UnrollMaxCount;
- if (UnrollMaxUpperBound.getNumOccurrences() > 0)
+ if (UnrollMaxUpperBound.getNumOccurrences() > 0) {
UP.MaxUpperBound = UnrollMaxUpperBound;
+ UP.MaxUpperBoundWithEarlyExits = UP.MaxUpperBound;
+ }
+ if (UnrollMaxUpperBoundWithEarlyExits.getNumOccurrences() > 0)
+ UP.MaxUpperBoundWithEarlyExits = UnrollMaxUpperBoundWithEarlyExits;
if (UnrollFullMaxCount.getNumOccurrences() > 0)
UP.FullUnrollMaxCount = UnrollFullMaxCount;
if (UnrollAllowPartial.getNumOccurrences() > 0)
@@ -1110,8 +1120,10 @@ unsigned llvm::computeUnrollCount(
// cost of exact full unrolling. As such, if we have an exact count and
// found it unprofitable, we'll never chose to bounded unroll.
LLVM_DEBUG(dbgs().indent(1) << "Trying upper-bound unroll...\n");
+ unsigned UpperBoundLimit =
+ MaxOrZero ? UP.MaxUpperBound : UP.MaxUpperBoundWithEarlyExits;
if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&
- MaxTripCount <= UP.MaxUpperBound) {
+ MaxTripCount <= UpperBoundLimit) {
if (auto UnrollFactor =
shouldFullUnroll(L, TTI, DT, SE, EphValues, MaxTripCount, UCE, UP))
return *UnrollFactor;
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
new file mode 100644
index 0000000000000..9a034f01d76d3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=DEFAULT
+; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-max-upperbound-with-early-exits=8 < %s | FileCheck %s --check-prefixes=WIDE
+
+; AArch64 uses a lower upper-bound limit for loops that can exit early. A typical
+; case is a varint-length helper:
+;
+; int varint_len(uint32_t serial_type) {
+; uint64_t v = serial_type;
+; int i;
+; for (i = 1; (v >>= 7) != 0; i++) {}
+; return i;
+; }
+;
+; This loop runs at most 6 times, over the AArch64 limit of 5, so it is not
+; unrolled by default. Raising the bound to 8 lets it unroll.
+define i32 @varint_i32(i32 %serial_type) {
+; DEFAULT-LABEL: @varint_i32(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; DEFAULT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; WIDE-LABEL: @varint_i32(
+; WIDE-NEXT: entry:
+; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; WIDE-NEXT: br label [[LOOP:%.*]]
+; WIDE: loop:
+; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; WIDE: loop.1:
+; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; WIDE: loop.2:
+; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; WIDE: loop.3:
+; WIDE-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
+; WIDE-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
+; WIDE-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
+; WIDE: loop.4:
+; WIDE-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
+; WIDE-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
+; WIDE-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
+; WIDE: loop.5:
+; WIDE-NEXT: br label [[EXIT]]
+; WIDE: exit:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; WIDE-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; The 16-bit version runs at most 4 times, under the limit of 5, so it is still
+; unrolled by default.
+define i32 @varint_i16(i16 %serial_type) {
+; DEFAULT-LABEL: @varint_i16(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; DEFAULT: loop.2:
+; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; DEFAULT: loop.3:
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; WIDE-LABEL: @varint_i16(
+; WIDE-NEXT: entry:
+; WIDE-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
+; WIDE-NEXT: br label [[LOOP:%.*]]
+; WIDE: loop:
+; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; WIDE: loop.1:
+; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; WIDE: loop.2:
+; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; WIDE: loop.3:
+; WIDE-NEXT: br label [[EXIT]]
+; WIDE: exit:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; WIDE-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i16 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
new file mode 100644
index 0000000000000..29ff1345cfd07
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes='loop-unroll<upperbound>' < %s | FileCheck %s --check-prefixes=DEFAULT
+; RUN: opt -S -passes='loop-unroll<upperbound>' -unroll-max-upperbound-with-early-exits=5 < %s | FileCheck %s --check-prefixes=LIMIT
+
+; -unroll-max-upperbound-with-early-exits only limits loops that can exit early.
+; Loops that run their max count or zero times still use -unroll-max-upperbound.
+
+; This loop can exit on any iteration and runs at most 6 times. The bound of 5
+; is below that, so it is not unrolled.
+define i32 @early_exit(i32 %serial_type) {
+; DEFAULT-LABEL: @early_exit(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; DEFAULT: loop.2:
+; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; DEFAULT: loop.3:
+; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
+; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
+; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
+; DEFAULT: loop.4:
+; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
+; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
+; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
+; DEFAULT: loop.5:
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; LIMIT-LABEL: @early_exit(
+; LIMIT-NEXT: entry:
+; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; LIMIT-NEXT: br label [[LOOP:%.*]]
+; LIMIT: loop:
+; LIMIT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; LIMIT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; LIMIT: exit:
+; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; LIMIT-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; This loop runs at most 6 times, but always its max count or zero times, so the
+; option does not affect it. It is fully unrolled in both runs.
+define i32 @max_or_zero(i32 %n, ptr %p) {
+; DEFAULT-LABEL: @max_or_zero(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; DEFAULT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
+; DEFAULT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
+; DEFAULT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
+; DEFAULT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
+; DEFAULT-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
+; DEFAULT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
+; DEFAULT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; LIMIT-LABEL: @max_or_zero(
+; LIMIT-NEXT: entry:
+; LIMIT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
+; LIMIT-NEXT: br label [[LOOP:%.*]]
+; LIMIT: loop:
+; LIMIT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; LIMIT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
+; LIMIT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
+; LIMIT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; LIMIT: loop.1:
+; LIMIT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
+; LIMIT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
+; LIMIT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
+; LIMIT-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
+; LIMIT-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
+; LIMIT-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
+; LIMIT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
+; LIMIT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
+; LIMIT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
+; LIMIT-NEXT: br label [[EXIT]]
+; LIMIT: exit:
+; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; LIMIT-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %end = add i32 %n, 6
+ br label %loop
+
+loop:
+ %i = phi i32 [ %n, %entry ], [ %i.next, %loop ]
+ store i32 %i, ptr %p
+ %i.next = add i32 %i, 1
+ %c = icmp ult i32 %i.next, %end
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret i32 %i
+}
>From d4b6a2c93303f91247704b1a86f5dfdf44e58488 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 24 Jun 2026 09:59:33 +0000
Subject: [PATCH 2/5] Rework into an opt-in unknown-exact-trip-count limit
Classify loop preciseily - single exiting block, !MaxOrZero and an exact
backedge count that is SCEVCouldNotTcompute.
---
.../llvm/Analysis/TargetTransformInfo.h | 9 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 2 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 51 ++++-
...nroll-max-upperbound-unknown-trip-count.ll | 98 +++++++++
.../unroll-max-upperbound-with-early-exits.ll | 142 ------------
...nroll-max-upperbound-unknown-trip-count.ll | 202 ++++++++++++++++++
.../unroll-max-upperbound-with-early-exits.ll | 137 ------------
7 files changed, 348 insertions(+), 293 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index c18b246f0d633..78b52843db559 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -697,9 +697,12 @@ class TargetTransformInfo {
/// to be overrided by a target gives more flexiblity on certain cases.
/// By default, MaxUpperBound uses UnrollMaxUpperBound which value is 8.
unsigned MaxUpperBound;
- /// Set the maximum upper bound of trip count for loops that may exit before
- /// reaching the bound. This defaults to MaxUpperBound.
- unsigned MaxUpperBoundWithEarlyExits;
+ /// Set a separate, lower maximum trip-count upper bound for loops whose
+ /// exact trip count is unknown but a small conservative maximum is known
+ /// (such as loops that exit at a runtime-determined iteration). Defaults to
+ /// 0, which disables this limit and leaves such loops on the normal
+ /// MaxUpperBound path; a target opts in by setting a small value.
+ unsigned MaxUpperBoundUnknownTripCount;
/// Set the maximum unrolling factor for full unrolling. Like MaxCount, but
/// applies even if full unrolling is selected. This allows a target to fall
/// back to Partial unrolling if full unrolling is above FullUnrollMaxCount.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index aa6e95d0ab816..de3d8f2f0690d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6041,7 +6041,7 @@ void AArch64TTIImpl::getUnrollingPreferences(
BaseT::getUnrollingPreferences(L, SE, UP, ORE);
UP.UpperBound = true;
- UP.MaxUpperBoundWithEarlyExits = 5;
+ UP.MaxUpperBoundUnknownTripCount = 5;
// For inner loop, it is more likely to be a hot one, and the runtime check
// can be promoted out from LICM pass, so the overhead is less, let's try
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 21374a6476db7..733757c1f9ce6 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -141,10 +141,11 @@ static cl::opt<unsigned> UnrollMaxUpperBound(
cl::desc(
"The max of trip count upper bound that is considered in unrolling"));
-static cl::opt<unsigned> UnrollMaxUpperBoundWithEarlyExits(
- "unroll-max-upperbound-with-early-exits", cl::Hidden,
+static cl::opt<unsigned> UnrollMaxUpperBoundUnknownTripCount(
+ "unroll-max-upperbound-unknown-trip-count", cl::Hidden,
cl::desc("The max of trip count upper bound that is considered in "
- "unrolling loops that may exit before reaching the bound"));
+ "unrolling loops whose exact trip count is unknown (only a "
+ "conservative maximum is known)"));
static cl::opt<unsigned> PragmaUnrollThreshold(
"pragma-unroll-threshold", cl::init(16 * 1024), cl::Hidden,
@@ -210,7 +211,9 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.DefaultUnrollRuntimeCount = 8;
UP.MaxCount = std::numeric_limits<unsigned>::max();
UP.MaxUpperBound = UnrollMaxUpperBound;
- UP.MaxUpperBoundWithEarlyExits = UP.MaxUpperBound;
+ // 0 means the unknown-exact-trip-count limit is disabled; a target (or the
+ // -unroll-max-upperbound-unknown-trip-count option) opts in by setting it.
+ UP.MaxUpperBoundUnknownTripCount = 0;
UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
UP.BEInsns = 2;
UP.Partial = false;
@@ -251,12 +254,10 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;
if (UnrollMaxCount.getNumOccurrences() > 0)
UP.MaxCount = UnrollMaxCount;
- if (UnrollMaxUpperBound.getNumOccurrences() > 0) {
+ if (UnrollMaxUpperBound.getNumOccurrences() > 0)
UP.MaxUpperBound = UnrollMaxUpperBound;
- UP.MaxUpperBoundWithEarlyExits = UP.MaxUpperBound;
- }
- if (UnrollMaxUpperBoundWithEarlyExits.getNumOccurrences() > 0)
- UP.MaxUpperBoundWithEarlyExits = UnrollMaxUpperBoundWithEarlyExits;
+ if (UnrollMaxUpperBoundUnknownTripCount.getNumOccurrences() > 0)
+ UP.MaxUpperBoundUnknownTripCount = UnrollMaxUpperBoundUnknownTripCount;
if (UnrollFullMaxCount.getNumOccurrences() > 0)
UP.FullUnrollMaxCount = UnrollFullMaxCount;
if (UnrollAllowPartial.getNumOccurrences() > 0)
@@ -1107,6 +1108,24 @@ unsigned llvm::computeUnrollCount(
return *UnrollFactor;
}
+ // A loop can have a known small maximum trip count while SCEV still cannot
+ // form an exact backedge count (getBackedgeTakenCount stays
+ // SCEVCouldNotCompute) - typically a data-dependent exit, e.g. shifting a
+ // value until it reaches zero. Unrolling such a loop replaces one
+ // well-predicted backedge with several rarely-taken exit branches - costing
+ // branch-predictor capacity and code size - while the data-dependent exit
+ // limits the usual unroll benefit. A target can therefore set
+ // MaxUpperBoundUnknownTripCount to hold these loops to a lower unroll bound
+ // than MaxUpperBound; the bounded and runtime routes below apply it.
+ //
+ // The check is off by default (a 0 limit skips it) and only applies to
+ // upper-bound unrolling (UP.UpperBound); MaxOrZero and multi-exit loops are
+ // excluded, and UP.Force overrides it.
+ const bool ApplyUnknownTripCountLimit =
+ UP.MaxUpperBoundUnknownTripCount && UP.UpperBound && !UP.Force &&
+ L->getExitingBlock() && !MaxOrZero &&
+ isa<SCEVCouldNotCompute>(SE.getBackedgeTakenCount(L));
+
// 4th priority is bounded unrolling.
// We can unroll by the upper bound amount if it's generally allowed or if
// we know that the loop is executed either the upper bound or zero times.
@@ -1121,7 +1140,9 @@ unsigned llvm::computeUnrollCount(
// found it unprofitable, we'll never chose to bounded unroll.
LLVM_DEBUG(dbgs().indent(1) << "Trying upper-bound unroll...\n");
unsigned UpperBoundLimit =
- MaxOrZero ? UP.MaxUpperBound : UP.MaxUpperBoundWithEarlyExits;
+ ApplyUnknownTripCountLimit
+ ? std::min(UP.MaxUpperBound, UP.MaxUpperBoundUnknownTripCount)
+ : UP.MaxUpperBound;
if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&
MaxTripCount <= UpperBoundLimit) {
if (auto UnrollFactor =
@@ -1169,6 +1190,16 @@ unsigned llvm::computeUnrollCount(
return 0;
}
+ // Also skip runtime unrolling when the exact trip count is unknown. That path
+ // may clamp the unroll count to the max trip count, effectively fully
+ // unrolling the loop.
+ if (MaxTripCount && ApplyUnknownTripCountLimit) {
+ LLVM_DEBUG(dbgs().indent(2)
+ << "Not runtime unrolling: max trip count " << MaxTripCount
+ << " has an unknown exact value, and not forced.\n");
+ return;
+ }
+
// Check if the runtime trip count is too small when profile is available.
if (L->getHeader()->getParent()->hasProfileData()) {
if (auto ProfileTripCount = getLoopEstimatedTripCount(L)) {
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
new file mode 100644
index 0000000000000..9626080709517
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s
+
+; AArch64 caps upper-bound unrolling of loops whose exact trip count is unknown
+; (only a conservative maximum is known) at a maximum trip count of 5: such a
+; loop is unrolled only if it may run at most 5 times. This file pins that
+; default on both sides of the boundary; the option mechanics are covered
+; target-independently in ../unroll-max-upperbound-unknown-trip-count.ll.
+;
+; A typical case is a varint-length helper:
+;
+; int varint_len(uint32_t serial_type) {
+; uint64_t v = serial_type;
+; int i;
+; for (i = 1; (v >>= 7) != 0; i++) {}
+; return i;
+; }
+;
+; This loop runs at most 6 times, over the AArch64 limit of 5, so it is not
+; unrolled by default.
+define i32 @varint_i32(i32 %serial_type) {
+;
+;
+; CHECK-LABEL: @varint_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; CHECK-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; The 16-bit version runs at most 4 times, under the limit of 5, so it is still
+; unrolled by default.
+define i32 @varint_i16(i16 %serial_type) {
+;
+;
+; CHECK-LABEL: @varint_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; CHECK: loop.1:
+; CHECK-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; CHECK-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; CHECK-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; CHECK: loop.2:
+; CHECK-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; CHECK-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; CHECK-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; CHECK: loop.3:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; CHECK-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i16 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
deleted file mode 100644
index 9a034f01d76d3..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-with-early-exits.ll
+++ /dev/null
@@ -1,142 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=DEFAULT
-; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-max-upperbound-with-early-exits=8 < %s | FileCheck %s --check-prefixes=WIDE
-
-; AArch64 uses a lower upper-bound limit for loops that can exit early. A typical
-; case is a varint-length helper:
-;
-; int varint_len(uint32_t serial_type) {
-; uint64_t v = serial_type;
-; int i;
-; for (i = 1; (v >>= 7) != 0; i++) {}
-; return i;
-; }
-;
-; This loop runs at most 6 times, over the AArch64 limit of 5, so it is not
-; unrolled by default. Raising the bound to 8 lets it unroll.
-define i32 @varint_i32(i32 %serial_type) {
-; DEFAULT-LABEL: @varint_i32(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
-; DEFAULT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
-; DEFAULT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; WIDE-LABEL: @varint_i32(
-; WIDE-NEXT: entry:
-; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; WIDE-NEXT: br label [[LOOP:%.*]]
-; WIDE: loop:
-; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; WIDE: loop.1:
-; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; WIDE: loop.2:
-; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; WIDE: loop.3:
-; WIDE-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
-; WIDE-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
-; WIDE-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
-; WIDE: loop.4:
-; WIDE-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
-; WIDE-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
-; WIDE-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
-; WIDE: loop.5:
-; WIDE-NEXT: br label [[EXIT]]
-; WIDE: exit:
-; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
-; WIDE-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i32 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
-
-; The 16-bit version runs at most 4 times, under the limit of 5, so it is still
-; unrolled by default.
-define i32 @varint_i16(i16 %serial_type) {
-; DEFAULT-LABEL: @varint_i16(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
-; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; DEFAULT: loop.2:
-; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; DEFAULT: loop.3:
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; WIDE-LABEL: @varint_i16(
-; WIDE-NEXT: entry:
-; WIDE-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
-; WIDE-NEXT: br label [[LOOP:%.*]]
-; WIDE: loop:
-; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; WIDE: loop.1:
-; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; WIDE: loop.2:
-; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; WIDE: loop.3:
-; WIDE-NEXT: br label [[EXIT]]
-; WIDE: exit:
-; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
-; WIDE-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i16 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
new file mode 100644
index 0000000000000..9a4dc839f89a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
@@ -0,0 +1,202 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes='loop-unroll<upperbound>' < %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -S -passes='loop-unroll<upperbound>' -unroll-max-upperbound-unknown-trip-count=5 < %s | FileCheck %s --check-prefixes=CHECK,LIMIT
+
+; -unroll-max-upperbound-unknown-trip-count only limits loops whose exact trip
+; count SCEV cannot compute (only a conservative maximum is known). Loops that
+; run their max count or zero times (MaxOrZero) still use -unroll-max-upperbound.
+
+; This loop's exact trip count is unknown - it exits at a runtime-determined
+; iteration - and it runs at most 6 times. The bound of 5 is below that, so it
+; is not unrolled.
+define i32 @unknown_trip_count(i32 %serial_type) {
+; DEFAULT-LABEL: @unknown_trip_count(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; DEFAULT: loop.2:
+; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; DEFAULT: loop.3:
+; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
+; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
+; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
+; DEFAULT: loop.4:
+; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
+; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
+; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
+; DEFAULT: loop.5:
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; LIMIT-LABEL: @unknown_trip_count(
+; LIMIT-NEXT: entry:
+; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; LIMIT-NEXT: br label [[LOOP:%.*]]
+; LIMIT: loop:
+; LIMIT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; LIMIT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; LIMIT: exit:
+; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; LIMIT-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; This loop runs at most 6 times, and - because %n + 6 may wrap - either 6 or 1
+; times (MaxOrZero), never in between, so the option does not affect it. It is
+; fully unrolled in both runs.
+define i32 @max_or_zero(i32 %n, ptr %p) {
+; CHECK-LABEL: @max_or_zero(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
+; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
+; CHECK-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; CHECK: loop.1:
+; CHECK-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
+; CHECK-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
+; CHECK-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
+; CHECK-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
+; CHECK-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
+; CHECK-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
+; CHECK-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
+; CHECK-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
+; CHECK-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; CHECK-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %end = add i32 %n, 6
+ br label %loop
+
+loop:
+ %i = phi i32 [ %n, %entry ], [ %i.next, %loop ]
+ store i32 %i, ptr %p
+ %i.next = add i32 %i, 1
+ %c = icmp ult i32 %i.next, %end
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret i32 %i
+}
+
+; With profile data showing the loop usually exits on the first iteration, the
+; limit does not leave the loop untouched: bounded unrolling is refused, and
+; peeling handles the common case instead. Without the limit the loop is
+; bounded-unrolled as usual.
+define i32 @peel_with_profile(i32 %serial_type) !prof !0 {
+; DEFAULT-LABEL: @peel_with_profile(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]], !prof [[PROF2:![0-9]+]]
+; DEFAULT: loop.2:
+; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]], !prof [[PROF2]]
+; DEFAULT: loop.3:
+; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
+; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
+; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]], !prof [[PROF2]]
+; DEFAULT: loop.4:
+; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
+; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
+; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]], !prof [[PROF2]]
+; DEFAULT: loop.5:
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; LIMIT-LABEL: @peel_with_profile(
+; LIMIT-NEXT: entry:
+; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; LIMIT-NEXT: br label [[LOOP_PEEL_BEGIN:%.*]]
+; LIMIT: loop.peel.begin:
+; LIMIT-NEXT: br label [[LOOP_PEEL:%.*]]
+; LIMIT: loop.peel:
+; LIMIT-NEXT: [[V_SHR_PEEL:%.*]] = lshr i64 [[V0]], 7
+; LIMIT-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 1, 1
+; LIMIT-NEXT: [[CMP_PEEL:%.*]] = icmp ne i64 [[V_SHR_PEEL]], 0
+; LIMIT-NEXT: br i1 [[CMP_PEEL]], label [[LOOP_PEEL_NEXT:%.*]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]
+; LIMIT: loop.peel.next:
+; LIMIT-NEXT: br label [[LOOP_PEEL_NEXT1:%.*]]
+; LIMIT: loop.peel.next1:
+; LIMIT-NEXT: br label [[ENTRY_PEEL_NEWPH:%.*]]
+; LIMIT: entry.peel.newph:
+; LIMIT-NEXT: br label [[LOOP:%.*]]
+; LIMIT: loop:
+; LIMIT-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V_SHR_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; LIMIT-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof [[PROF1]], !llvm.loop [[LOOP2:![0-9]+]]
+; LIMIT: exit.loopexit:
+; LIMIT-NEXT: [[I_LCSSA_PH:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; LIMIT-NEXT: br label [[EXIT]]
+; LIMIT: exit:
+; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP_PEEL]] ], [ [[I_LCSSA_PH]], [[EXIT_LOOPEXIT]] ]
+; LIMIT-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit, !prof !1
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+!0 = !{!"function_entry_count", i64 1000}
+!1 = !{!"branch_weights", i32 300, i32 1000}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
deleted file mode 100644
index 29ff1345cfd07..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-with-early-exits.ll
+++ /dev/null
@@ -1,137 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes='loop-unroll<upperbound>' < %s | FileCheck %s --check-prefixes=DEFAULT
-; RUN: opt -S -passes='loop-unroll<upperbound>' -unroll-max-upperbound-with-early-exits=5 < %s | FileCheck %s --check-prefixes=LIMIT
-
-; -unroll-max-upperbound-with-early-exits only limits loops that can exit early.
-; Loops that run their max count or zero times still use -unroll-max-upperbound.
-
-; This loop can exit on any iteration and runs at most 6 times. The bound of 5
-; is below that, so it is not unrolled.
-define i32 @early_exit(i32 %serial_type) {
-; DEFAULT-LABEL: @early_exit(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
-; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; DEFAULT: loop.2:
-; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; DEFAULT: loop.3:
-; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
-; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
-; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
-; DEFAULT: loop.4:
-; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
-; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
-; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
-; DEFAULT: loop.5:
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; LIMIT-LABEL: @early_exit(
-; LIMIT-NEXT: entry:
-; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; LIMIT-NEXT: br label [[LOOP:%.*]]
-; LIMIT: loop:
-; LIMIT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
-; LIMIT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; LIMIT: exit:
-; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
-; LIMIT-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i32 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
-
-; This loop runs at most 6 times, but always its max count or zero times, so the
-; option does not affect it. It is fully unrolled in both runs.
-define i32 @max_or_zero(i32 %n, ptr %p) {
-; DEFAULT-LABEL: @max_or_zero(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
-; DEFAULT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
-; DEFAULT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
-; DEFAULT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
-; DEFAULT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
-; DEFAULT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
-; DEFAULT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
-; DEFAULT-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
-; DEFAULT-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
-; DEFAULT-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
-; DEFAULT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
-; DEFAULT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
-; DEFAULT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; LIMIT-LABEL: @max_or_zero(
-; LIMIT-NEXT: entry:
-; LIMIT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
-; LIMIT-NEXT: br label [[LOOP:%.*]]
-; LIMIT: loop:
-; LIMIT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
-; LIMIT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
-; LIMIT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
-; LIMIT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; LIMIT: loop.1:
-; LIMIT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
-; LIMIT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
-; LIMIT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
-; LIMIT-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
-; LIMIT-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
-; LIMIT-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
-; LIMIT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
-; LIMIT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
-; LIMIT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
-; LIMIT-NEXT: br label [[EXIT]]
-; LIMIT: exit:
-; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
-; LIMIT-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %end = add i32 %n, 6
- br label %loop
-
-loop:
- %i = phi i32 [ %n, %entry ], [ %i.next, %loop ]
- store i32 %i, ptr %p
- %i.next = add i32 %i, 1
- %c = icmp ult i32 %i.next, %end
- br i1 %c, label %loop, label %exit
-
-exit:
- ret i32 %i
-}
>From 0fab9351060772926b1f032462abe369fa8b40ba Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 7 Jul 2026 10:38:44 +0000
Subject: [PATCH 3/5] Rework as a target-local AArch64 limit
---
.../llvm/Analysis/TargetTransformInfo.h | 6 -
.../AArch64/AArch64TargetTransformInfo.cpp | 18 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 45 +---
...-max-upperbound-uncomputable-trip-count.ll | 211 ++++++++++++++++++
...nroll-max-upperbound-unknown-trip-count.ll | 98 --------
...nroll-max-upperbound-unknown-trip-count.ll | 202 -----------------
6 files changed, 229 insertions(+), 351 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 78b52843db559..107ae4dba5075 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -697,12 +697,6 @@ class TargetTransformInfo {
/// to be overrided by a target gives more flexiblity on certain cases.
/// By default, MaxUpperBound uses UnrollMaxUpperBound which value is 8.
unsigned MaxUpperBound;
- /// Set a separate, lower maximum trip-count upper bound for loops whose
- /// exact trip count is unknown but a small conservative maximum is known
- /// (such as loops that exit at a runtime-determined iteration). Defaults to
- /// 0, which disables this limit and leaves such loops on the normal
- /// MaxUpperBound path; a target opts in by setting a small value.
- unsigned MaxUpperBoundUnknownTripCount;
/// Set the maximum unrolling factor for full unrolling. Like MaxCount, but
/// applies even if full unrolling is selected. This allows a target to fall
/// back to Partial unrolling if full unrolling is above FullUnrollMaxCount.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index de3d8f2f0690d..7ed017a4ec349 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6041,7 +6041,23 @@ void AArch64TTIImpl::getUnrollingPreferences(
BaseT::getUnrollingPreferences(L, SE, UP, ORE);
UP.UpperBound = true;
- UP.MaxUpperBoundUnknownTripCount = 5;
+
+ // A loop can have a known small maximum trip count while SCEV still cannot
+ // form an exact backedge count (getBackedgeTakenCount stays
+ // SCEVCouldNotCompute) - typically a data-dependent exit, e.g. shifting a
+ // value until it reaches zero. Unrolling such a loop replaces one
+ // well-predicted backedge with several rarely-taken exit branches - costing
+ // branch-predictor capacity and code size - while the data-dependent exit
+ // limits the usual unroll benefit. Hold these loops to a lower upper bound
+ // than the default MaxUpperBound; 5 still lets smaller early-exit loops
+ // unroll. Also disable runtime unrolling: with a known small maximum trip
+ // count it would clamp the unroll count to that maximum and turn into the
+ // same complete unroll.
+ if (L->getExitingBlock() && !SE.isBackedgeTakenCountMaxOrZero(L) &&
+ isa<SCEVCouldNotCompute>(SE.getBackedgeTakenCount(L))) {
+ UP.MaxUpperBound = 5;
+ UP.Runtime = false;
+ }
// For inner loop, it is more likely to be a hot one, and the runtime check
// can be promoted out from LICM pass, so the overhead is less, let's try
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 733757c1f9ce6..ad3b123f3327c 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -141,12 +141,6 @@ static cl::opt<unsigned> UnrollMaxUpperBound(
cl::desc(
"The max of trip count upper bound that is considered in unrolling"));
-static cl::opt<unsigned> UnrollMaxUpperBoundUnknownTripCount(
- "unroll-max-upperbound-unknown-trip-count", cl::Hidden,
- cl::desc("The max of trip count upper bound that is considered in "
- "unrolling loops whose exact trip count is unknown (only a "
- "conservative maximum is known)"));
-
static cl::opt<unsigned> PragmaUnrollThreshold(
"pragma-unroll-threshold", cl::init(16 * 1024), cl::Hidden,
cl::desc("Unrolled size limit for loops with unroll metadata "
@@ -211,9 +205,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.DefaultUnrollRuntimeCount = 8;
UP.MaxCount = std::numeric_limits<unsigned>::max();
UP.MaxUpperBound = UnrollMaxUpperBound;
- // 0 means the unknown-exact-trip-count limit is disabled; a target (or the
- // -unroll-max-upperbound-unknown-trip-count option) opts in by setting it.
- UP.MaxUpperBoundUnknownTripCount = 0;
UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
UP.BEInsns = 2;
UP.Partial = false;
@@ -256,8 +247,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.MaxCount = UnrollMaxCount;
if (UnrollMaxUpperBound.getNumOccurrences() > 0)
UP.MaxUpperBound = UnrollMaxUpperBound;
- if (UnrollMaxUpperBoundUnknownTripCount.getNumOccurrences() > 0)
- UP.MaxUpperBoundUnknownTripCount = UnrollMaxUpperBoundUnknownTripCount;
if (UnrollFullMaxCount.getNumOccurrences() > 0)
UP.FullUnrollMaxCount = UnrollFullMaxCount;
if (UnrollAllowPartial.getNumOccurrences() > 0)
@@ -1108,24 +1097,6 @@ unsigned llvm::computeUnrollCount(
return *UnrollFactor;
}
- // A loop can have a known small maximum trip count while SCEV still cannot
- // form an exact backedge count (getBackedgeTakenCount stays
- // SCEVCouldNotCompute) - typically a data-dependent exit, e.g. shifting a
- // value until it reaches zero. Unrolling such a loop replaces one
- // well-predicted backedge with several rarely-taken exit branches - costing
- // branch-predictor capacity and code size - while the data-dependent exit
- // limits the usual unroll benefit. A target can therefore set
- // MaxUpperBoundUnknownTripCount to hold these loops to a lower unroll bound
- // than MaxUpperBound; the bounded and runtime routes below apply it.
- //
- // The check is off by default (a 0 limit skips it) and only applies to
- // upper-bound unrolling (UP.UpperBound); MaxOrZero and multi-exit loops are
- // excluded, and UP.Force overrides it.
- const bool ApplyUnknownTripCountLimit =
- UP.MaxUpperBoundUnknownTripCount && UP.UpperBound && !UP.Force &&
- L->getExitingBlock() && !MaxOrZero &&
- isa<SCEVCouldNotCompute>(SE.getBackedgeTakenCount(L));
-
// 4th priority is bounded unrolling.
// We can unroll by the upper bound amount if it's generally allowed or if
// we know that the loop is executed either the upper bound or zero times.
@@ -1139,12 +1110,8 @@ unsigned llvm::computeUnrollCount(
// cost of exact full unrolling. As such, if we have an exact count and
// found it unprofitable, we'll never chose to bounded unroll.
LLVM_DEBUG(dbgs().indent(1) << "Trying upper-bound unroll...\n");
- unsigned UpperBoundLimit =
- ApplyUnknownTripCountLimit
- ? std::min(UP.MaxUpperBound, UP.MaxUpperBoundUnknownTripCount)
- : UP.MaxUpperBound;
if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&
- MaxTripCount <= UpperBoundLimit) {
+ MaxTripCount <= UP.MaxUpperBound) {
if (auto UnrollFactor =
shouldFullUnroll(L, TTI, DT, SE, EphValues, MaxTripCount, UCE, UP))
return *UnrollFactor;
@@ -1190,16 +1157,6 @@ unsigned llvm::computeUnrollCount(
return 0;
}
- // Also skip runtime unrolling when the exact trip count is unknown. That path
- // may clamp the unroll count to the max trip count, effectively fully
- // unrolling the loop.
- if (MaxTripCount && ApplyUnknownTripCountLimit) {
- LLVM_DEBUG(dbgs().indent(2)
- << "Not runtime unrolling: max trip count " << MaxTripCount
- << " has an unknown exact value, and not forced.\n");
- return;
- }
-
// Check if the runtime trip count is too small when profile is available.
if (L->getHeader()->getParent()->hasProfileData()) {
if (auto ProfileTripCount = getLoopEstimatedTripCount(L)) {
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
new file mode 100644
index 0000000000000..fd266c93d6838
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=DEFAULT
+; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-max-upperbound=8 < %s | FileCheck %s --check-prefixes=WIDE
+
+; AArch64 caps upper-bound unrolling at a max trip count of 5 for loops whose
+; exact trip count is not computable (data-dependent exits). A typical case is
+; a varint-length helper:
+;
+; int varint_len(uint32_t serial_type) {
+; uint64_t v = serial_type;
+; int i;
+; for (i = 1; (v >>= 7) != 0; i++) {}
+; return i;
+; }
+;
+; This loop runs at most 6 times, over the limit of 5, so it is not unrolled
+; by default. -unroll-max-upperbound is applied after the target preferences,
+; so raising it to 8 lets the loop unroll again.
+define i32 @varint_i32(i32 %serial_type) {
+; DEFAULT-LABEL: @varint_i32(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; DEFAULT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
+; DEFAULT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; WIDE-LABEL: @varint_i32(
+; WIDE-NEXT: entry:
+; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
+; WIDE-NEXT: br label [[LOOP:%.*]]
+; WIDE: loop:
+; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; WIDE: loop.1:
+; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; WIDE: loop.2:
+; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; WIDE: loop.3:
+; WIDE-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
+; WIDE-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
+; WIDE-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
+; WIDE: loop.4:
+; WIDE-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
+; WIDE-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
+; WIDE-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
+; WIDE: loop.5:
+; WIDE-NEXT: br label [[EXIT]]
+; WIDE: exit:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; WIDE-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i32 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; The 16-bit version runs at most 4 times, under the limit of 5, so it is
+; still unrolled by default.
+define i32 @varint_i16(i16 %serial_type) {
+; DEFAULT-LABEL: @varint_i16(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; DEFAULT: loop.2:
+; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; DEFAULT: loop.3:
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; WIDE-LABEL: @varint_i16(
+; WIDE-NEXT: entry:
+; WIDE-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
+; WIDE-NEXT: br label [[LOOP:%.*]]
+; WIDE: loop:
+; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
+; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
+; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; WIDE: loop.1:
+; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
+; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
+; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
+; WIDE: loop.2:
+; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
+; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
+; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
+; WIDE: loop.3:
+; WIDE-NEXT: br label [[EXIT]]
+; WIDE: exit:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; WIDE-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %v0 = zext i16 %serial_type to i64
+ br label %loop
+
+loop:
+ %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
+ %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
+ %v.shr = lshr i64 %v, 7
+ %i.next = add i32 %i, 1
+ %cmp = icmp ne i64 %v.shr, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %i.lcssa = phi i32 [ %i, %loop ]
+ ret i32 %i.lcssa
+}
+
+; This loop runs at most 6 times, and - because %n + 6 may wrap - either 6 or 1
+; times (MaxOrZero), never in between, so the option does not affect it. It is
+; fully unrolled in both runs.
+define i32 @max_or_zero(i32 %n, ptr %p) {
+; DEFAULT-LABEL: @max_or_zero(
+; DEFAULT-NEXT: entry:
+; DEFAULT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
+; DEFAULT-NEXT: br label [[LOOP:%.*]]
+; DEFAULT: loop:
+; DEFAULT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; DEFAULT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
+; DEFAULT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
+; DEFAULT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; DEFAULT: loop.1:
+; DEFAULT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
+; DEFAULT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
+; DEFAULT-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
+; DEFAULT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
+; DEFAULT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
+; DEFAULT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
+; DEFAULT-NEXT: br label [[EXIT]]
+; DEFAULT: exit:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
+;
+; WIDE-LABEL: @max_or_zero(
+; WIDE-NEXT: entry:
+; WIDE-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
+; WIDE-NEXT: br label [[LOOP:%.*]]
+; WIDE: loop:
+; WIDE-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; WIDE-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
+; WIDE-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
+; WIDE-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
+; WIDE: loop.1:
+; WIDE-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
+; WIDE-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
+; WIDE-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
+; WIDE-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
+; WIDE-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
+; WIDE-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
+; WIDE-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
+; WIDE-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
+; WIDE-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
+; WIDE-NEXT: br label [[EXIT]]
+; WIDE: exit:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; WIDE-NEXT: ret i32 [[I_LCSSA]]
+;
+entry:
+ %end = add i32 %n, 6
+ br label %loop
+
+loop:
+ %i = phi i32 [ %n, %entry ], [ %i.next, %loop ]
+ store i32 %i, ptr %p
+ %i.next = add i32 %i, 1
+ %c = icmp ult i32 %i.next, %end
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret i32 %i
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
deleted file mode 100644
index 9626080709517..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-unknown-trip-count.ll
+++ /dev/null
@@ -1,98 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s
-
-; AArch64 caps upper-bound unrolling of loops whose exact trip count is unknown
-; (only a conservative maximum is known) at a maximum trip count of 5: such a
-; loop is unrolled only if it may run at most 5 times. This file pins that
-; default on both sides of the boundary; the option mechanics are covered
-; target-independently in ../unroll-max-upperbound-unknown-trip-count.ll.
-;
-; A typical case is a varint-length helper:
-;
-; int varint_len(uint32_t serial_type) {
-; uint64_t v = serial_type;
-; int i;
-; for (i = 1; (v >>= 7) != 0; i++) {}
-; return i;
-; }
-;
-; This loop runs at most 6 times, over the AArch64 limit of 5, so it is not
-; unrolled by default.
-define i32 @varint_i32(i32 %serial_type) {
-;
-;
-; CHECK-LABEL: @varint_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK: exit:
-; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
-; CHECK-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i32 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
-
-; The 16-bit version runs at most 4 times, under the limit of 5, so it is still
-; unrolled by default.
-define i32 @varint_i16(i16 %serial_type) {
-;
-;
-; CHECK-LABEL: @varint_i16(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
-; CHECK-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; CHECK: loop.1:
-; CHECK-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; CHECK-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; CHECK-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; CHECK: loop.2:
-; CHECK-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; CHECK-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; CHECK-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; CHECK: loop.3:
-; CHECK-NEXT: br label [[EXIT]]
-; CHECK: exit:
-; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
-; CHECK-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i16 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
deleted file mode 100644
index 9a4dc839f89a0..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/unroll-max-upperbound-unknown-trip-count.ll
+++ /dev/null
@@ -1,202 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes='loop-unroll<upperbound>' < %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
-; RUN: opt -S -passes='loop-unroll<upperbound>' -unroll-max-upperbound-unknown-trip-count=5 < %s | FileCheck %s --check-prefixes=CHECK,LIMIT
-
-; -unroll-max-upperbound-unknown-trip-count only limits loops whose exact trip
-; count SCEV cannot compute (only a conservative maximum is known). Loops that
-; run their max count or zero times (MaxOrZero) still use -unroll-max-upperbound.
-
-; This loop's exact trip count is unknown - it exits at a runtime-determined
-; iteration - and it runs at most 6 times. The bound of 5 is below that, so it
-; is not unrolled.
-define i32 @unknown_trip_count(i32 %serial_type) {
-; DEFAULT-LABEL: @unknown_trip_count(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
-; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; DEFAULT: loop.2:
-; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; DEFAULT: loop.3:
-; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
-; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
-; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
-; DEFAULT: loop.4:
-; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
-; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
-; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
-; DEFAULT: loop.5:
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; LIMIT-LABEL: @unknown_trip_count(
-; LIMIT-NEXT: entry:
-; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; LIMIT-NEXT: br label [[LOOP:%.*]]
-; LIMIT: loop:
-; LIMIT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
-; LIMIT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; LIMIT: exit:
-; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
-; LIMIT-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i32 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
-
-; This loop runs at most 6 times, and - because %n + 6 may wrap - either 6 or 1
-; times (MaxOrZero), never in between, so the option does not affect it. It is
-; fully unrolled in both runs.
-define i32 @max_or_zero(i32 %n, ptr %p) {
-; CHECK-LABEL: @max_or_zero(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
-; CHECK-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
-; CHECK-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
-; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; CHECK: loop.1:
-; CHECK-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
-; CHECK-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
-; CHECK-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
-; CHECK-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3
-; CHECK-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4
-; CHECK-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4
-; CHECK-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
-; CHECK-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
-; CHECK-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
-; CHECK-NEXT: br label [[EXIT]]
-; CHECK: exit:
-; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
-; CHECK-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %end = add i32 %n, 6
- br label %loop
-
-loop:
- %i = phi i32 [ %n, %entry ], [ %i.next, %loop ]
- store i32 %i, ptr %p
- %i.next = add i32 %i, 1
- %c = icmp ult i32 %i.next, %end
- br i1 %c, label %loop, label %exit
-
-exit:
- ret i32 %i
-}
-
-; With profile data showing the loop usually exits on the first iteration, the
-; limit does not leave the loop untouched: bounded unrolling is refused, and
-; peeling handles the common case instead. Without the limit the loop is
-; bounded-unrolled as usual.
-define i32 @peel_with_profile(i32 %serial_type) !prof !0 {
-; DEFAULT-LABEL: @peel_with_profile(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
-; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]
-; DEFAULT: loop.1:
-; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
-; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]], !prof [[PROF2:![0-9]+]]
-; DEFAULT: loop.2:
-; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
-; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]], !prof [[PROF2]]
-; DEFAULT: loop.3:
-; DEFAULT-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
-; DEFAULT-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
-; DEFAULT-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]], !prof [[PROF2]]
-; DEFAULT: loop.4:
-; DEFAULT-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
-; DEFAULT-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
-; DEFAULT-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]], !prof [[PROF2]]
-; DEFAULT: loop.5:
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
-; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
-;
-; LIMIT-LABEL: @peel_with_profile(
-; LIMIT-NEXT: entry:
-; LIMIT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; LIMIT-NEXT: br label [[LOOP_PEEL_BEGIN:%.*]]
-; LIMIT: loop.peel.begin:
-; LIMIT-NEXT: br label [[LOOP_PEEL:%.*]]
-; LIMIT: loop.peel:
-; LIMIT-NEXT: [[V_SHR_PEEL:%.*]] = lshr i64 [[V0]], 7
-; LIMIT-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 1, 1
-; LIMIT-NEXT: [[CMP_PEEL:%.*]] = icmp ne i64 [[V_SHR_PEEL]], 0
-; LIMIT-NEXT: br i1 [[CMP_PEEL]], label [[LOOP_PEEL_NEXT:%.*]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]
-; LIMIT: loop.peel.next:
-; LIMIT-NEXT: br label [[LOOP_PEEL_NEXT1:%.*]]
-; LIMIT: loop.peel.next1:
-; LIMIT-NEXT: br label [[ENTRY_PEEL_NEWPH:%.*]]
-; LIMIT: entry.peel.newph:
-; LIMIT-NEXT: br label [[LOOP:%.*]]
-; LIMIT: loop:
-; LIMIT-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V:%.*]] = phi i64 [ [[V_SHR_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
-; LIMIT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
-; LIMIT-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1
-; LIMIT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; LIMIT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof [[PROF1]], !llvm.loop [[LOOP2:![0-9]+]]
-; LIMIT: exit.loopexit:
-; LIMIT-NEXT: [[I_LCSSA_PH:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
-; LIMIT-NEXT: br label [[EXIT]]
-; LIMIT: exit:
-; LIMIT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP_PEEL]] ], [ [[I_LCSSA_PH]], [[EXIT_LOOPEXIT]] ]
-; LIMIT-NEXT: ret i32 [[I_LCSSA]]
-;
-entry:
- %v0 = zext i32 %serial_type to i64
- br label %loop
-
-loop:
- %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]
- %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ]
- %v.shr = lshr i64 %v, 7
- %i.next = add i32 %i, 1
- %cmp = icmp ne i64 %v.shr, 0
- br i1 %cmp, label %loop, label %exit, !prof !1
-
-exit:
- %i.lcssa = phi i32 [ %i, %loop ]
- ret i32 %i.lcssa
-}
-
-!0 = !{!"function_entry_count", i64 1000}
-!1 = !{!"branch_weights", i32 300, i32 1000}
>From efbac4546a310a043294fb1098363463877fe236 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 31 Jul 2026 17:20:39 +0100
Subject: [PATCH 4/5] Address comments, update tests
---
.../AArch64/AArch64TargetTransformInfo.cpp | 18 +-
...-max-upperbound-uncomputable-trip-count.ll | 164 +++++++++---------
.../AArch64/induction-costs-sve.ll | 4 +-
.../AArch64/interleave-with-gaps.ll | 4 +-
.../AArch64/interleaving-load-store.ll | 4 +-
.../AArch64/reduction-recurrence-costs-sve.ll | 8 +-
.../LoopVectorize/AArch64/store-costs-sve.ll | 4 +-
.../AArch64/sve-epilog-vscale-fixed.ll | 4 +-
.../AArch64/sve2-histcnt-epilogue.ll | 4 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 4 +-
...row-interleave-to-widen-memory-scalable.ll | 4 +-
...form-narrow-interleave-vscale-x-UF-step.ll | 4 +-
.../LoopVectorize/AArch64/vector-reverse.ll | 4 +-
13 files changed, 104 insertions(+), 126 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7ed017a4ec349..9098eb05908ca 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6042,17 +6042,13 @@ void AArch64TTIImpl::getUnrollingPreferences(
UP.UpperBound = true;
- // A loop can have a known small maximum trip count while SCEV still cannot
- // form an exact backedge count (getBackedgeTakenCount stays
- // SCEVCouldNotCompute) - typically a data-dependent exit, e.g. shifting a
- // value until it reaches zero. Unrolling such a loop replaces one
- // well-predicted backedge with several rarely-taken exit branches - costing
- // branch-predictor capacity and code size - while the data-dependent exit
- // limits the usual unroll benefit. Hold these loops to a lower upper bound
- // than the default MaxUpperBound; 5 still lets smaller early-exit loops
- // unroll. Also disable runtime unrolling: with a known small maximum trip
- // count it would clamp the unroll count to that maximum and turn into the
- // same complete unroll.
+ // A loop can have a small maximum trip count while SCEV still cannot
+ // form an exact backedge count - typically a data-dependent exit, e.g.
+ // shifting a value until it reaches zero. Unrolling such a loop trades one
+ // well-predicted backedge for a chain of rarely-taken exit branches, so hold
+ // it to a lower upper bound; 5 still lets smaller early-exit loops unroll.
+ // Also disable runtime unrolling, which would clamp the unroll count to the
+ // known maximum trip count and produce the same complete unroll.
if (L->getExitingBlock() && !SE.isBackedgeTakenCountMaxOrZero(L) &&
isa<SCEVCouldNotCompute>(SE.getBackedgeTakenCount(L))) {
UP.MaxUpperBound = 5;
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
index fd266c93d6838..ec7b18792dad5 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=DEFAULT
; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-max-upperbound=8 < %s | FileCheck %s --check-prefixes=WIDE
@@ -17,49 +17,51 @@
; by default. -unroll-max-upperbound is applied after the target preferences,
; so raising it to 8 lets the loop unroll again.
define i32 @varint_i32(i32 %serial_type) {
-; DEFAULT-LABEL: @varint_i32(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], [[ENTRY]] ], [ [[V_SHR:%.*]], [[LOOP]] ]
+; DEFAULT-LABEL: define i32 @varint_i32(
+; DEFAULT-SAME: i32 [[SERIAL_TYPE:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE]] to i64
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], %[[ENTRY]] ], [ [[V_SHR:%.*]], %[[LOOP]] ]
; DEFAULT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7
; DEFAULT-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP]] ]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], %[[LOOP]] ]
; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
;
-; WIDE-LABEL: @varint_i32(
-; WIDE-NEXT: entry:
-; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE:%.*]] to i64
-; WIDE-NEXT: br label [[LOOP:%.*]]
-; WIDE: loop:
+; WIDE-LABEL: define i32 @varint_i32(
+; WIDE-SAME: i32 [[SERIAL_TYPE:%.*]]) {
+; WIDE-NEXT: [[ENTRY:.*:]]
+; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE]] to i64
+; WIDE-NEXT: br label %[[LOOP:.*]]
+; WIDE: [[LOOP]]:
; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; WIDE: loop.1:
+; WIDE-NEXT: br i1 [[CMP]], label %[[LOOP_1:.*]], label %[[EXIT:.*]]
+; WIDE: [[LOOP_1]]:
; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; WIDE: loop.2:
+; WIDE-NEXT: br i1 [[CMP_1]], label %[[LOOP_2:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_2]]:
; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; WIDE: loop.3:
+; WIDE-NEXT: br i1 [[CMP_2]], label %[[LOOP_3:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_3]]:
; WIDE-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7
; WIDE-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0
-; WIDE-NEXT: br i1 [[CMP_3]], label [[LOOP_4:%.*]], label [[EXIT]]
-; WIDE: loop.4:
+; WIDE-NEXT: br i1 [[CMP_3]], label %[[LOOP_4:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_4]]:
; WIDE-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7
; WIDE-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0
-; WIDE-NEXT: br i1 [[CMP_4]], label [[LOOP_5:%.*]], label [[EXIT]]
-; WIDE: loop.5:
-; WIDE-NEXT: br label [[EXIT]]
-; WIDE: exit:
-; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ], [ 5, [[LOOP_4]] ], [ 6, [[LOOP_5]] ]
+; WIDE-NEXT: br i1 [[CMP_4]], label %[[LOOP_5:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_5]]:
+; WIDE-NEXT: br label %[[EXIT]]
+; WIDE: [[EXIT]]:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, %[[LOOP]] ], [ 2, %[[LOOP_1]] ], [ 3, %[[LOOP_2]] ], [ 4, %[[LOOP_3]] ], [ 5, %[[LOOP_4]] ], [ 6, %[[LOOP_5]] ]
; WIDE-NEXT: ret i32 [[I_LCSSA]]
;
entry:
@@ -82,48 +84,50 @@ exit:
; The 16-bit version runs at most 4 times, under the limit of 5, so it is
; still unrolled by default.
define i32 @varint_i16(i16 %serial_type) {
-; DEFAULT-LABEL: @varint_i16(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
+; DEFAULT-LABEL: define i32 @varint_i16(
+; DEFAULT-SAME: i16 [[SERIAL_TYPE:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE]] to i64
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; DEFAULT-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[LOOP_1:.*]], label %[[EXIT:.*]]
+; DEFAULT: [[LOOP_1]]:
; DEFAULT-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
; DEFAULT-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; DEFAULT-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; DEFAULT: loop.2:
+; DEFAULT-NEXT: br i1 [[CMP_1]], label %[[LOOP_2:.*]], label %[[EXIT]]
+; DEFAULT: [[LOOP_2]]:
; DEFAULT-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
; DEFAULT-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; DEFAULT-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; DEFAULT: loop.3:
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; DEFAULT-NEXT: br i1 [[CMP_2]], label %[[LOOP_3:.*]], label %[[EXIT]]
+; DEFAULT: [[LOOP_3]]:
+; DEFAULT-NEXT: br label %[[EXIT]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, %[[LOOP]] ], [ 2, %[[LOOP_1]] ], [ 3, %[[LOOP_2]] ], [ 4, %[[LOOP_3]] ]
; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
;
-; WIDE-LABEL: @varint_i16(
-; WIDE-NEXT: entry:
-; WIDE-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE:%.*]] to i64
-; WIDE-NEXT: br label [[LOOP:%.*]]
-; WIDE: loop:
+; WIDE-LABEL: define i32 @varint_i16(
+; WIDE-SAME: i16 [[SERIAL_TYPE:%.*]]) {
+; WIDE-NEXT: [[ENTRY:.*:]]
+; WIDE-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE]] to i64
+; WIDE-NEXT: br label %[[LOOP:.*]]
+; WIDE: [[LOOP]]:
; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7
; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0
-; WIDE-NEXT: br i1 [[CMP]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; WIDE: loop.1:
+; WIDE-NEXT: br i1 [[CMP]], label %[[LOOP_1:.*]], label %[[EXIT:.*]]
+; WIDE: [[LOOP_1]]:
; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7
; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0
-; WIDE-NEXT: br i1 [[CMP_1]], label [[LOOP_2:%.*]], label [[EXIT]]
-; WIDE: loop.2:
+; WIDE-NEXT: br i1 [[CMP_1]], label %[[LOOP_2:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_2]]:
; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7
; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0
-; WIDE-NEXT: br i1 [[CMP_2]], label [[LOOP_3:%.*]], label [[EXIT]]
-; WIDE: loop.3:
-; WIDE-NEXT: br label [[EXIT]]
-; WIDE: exit:
-; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, [[LOOP]] ], [ 2, [[LOOP_1]] ], [ 3, [[LOOP_2]] ], [ 4, [[LOOP_3]] ]
+; WIDE-NEXT: br i1 [[CMP_2]], label %[[LOOP_3:.*]], label %[[EXIT]]
+; WIDE: [[LOOP_3]]:
+; WIDE-NEXT: br label %[[EXIT]]
+; WIDE: [[EXIT]]:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, %[[LOOP]] ], [ 2, %[[LOOP_1]] ], [ 3, %[[LOOP_2]] ], [ 4, %[[LOOP_3]] ]
; WIDE-NEXT: ret i32 [[I_LCSSA]]
;
entry:
@@ -147,16 +151,17 @@ exit:
; times (MaxOrZero), never in between, so the option does not affect it. It is
; fully unrolled in both runs.
define i32 @max_or_zero(i32 %n, ptr %p) {
-; DEFAULT-LABEL: @max_or_zero(
-; DEFAULT-NEXT: entry:
-; DEFAULT-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
-; DEFAULT-NEXT: br label [[LOOP:%.*]]
-; DEFAULT: loop:
-; DEFAULT-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; DEFAULT-LABEL: define i32 @max_or_zero(
+; DEFAULT-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[END:%.*]] = add i32 [[N]], 6
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: store i32 [[N]], ptr [[P]], align 4
; DEFAULT-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
; DEFAULT-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
-; DEFAULT-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; DEFAULT: loop.1:
+; DEFAULT-NEXT: br i1 [[C]], label %[[LOOP_1:.*]], label %[[EXIT:.*]]
+; DEFAULT: [[LOOP_1]]:
; DEFAULT-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
; DEFAULT-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
; DEFAULT-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
@@ -166,21 +171,22 @@ define i32 @max_or_zero(i32 %n, ptr %p) {
; DEFAULT-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
; DEFAULT-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
; DEFAULT-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
-; DEFAULT-NEXT: br label [[EXIT]]
-; DEFAULT: exit:
-; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; DEFAULT-NEXT: br label %[[EXIT]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], %[[LOOP]] ], [ [[I_NEXT_4]], %[[LOOP_1]] ]
; DEFAULT-NEXT: ret i32 [[I_LCSSA]]
;
-; WIDE-LABEL: @max_or_zero(
-; WIDE-NEXT: entry:
-; WIDE-NEXT: [[END:%.*]] = add i32 [[N:%.*]], 6
-; WIDE-NEXT: br label [[LOOP:%.*]]
-; WIDE: loop:
-; WIDE-NEXT: store i32 [[N]], ptr [[P:%.*]], align 4
+; WIDE-LABEL: define i32 @max_or_zero(
+; WIDE-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; WIDE-NEXT: [[ENTRY:.*:]]
+; WIDE-NEXT: [[END:%.*]] = add i32 [[N]], 6
+; WIDE-NEXT: br label %[[LOOP:.*]]
+; WIDE: [[LOOP]]:
+; WIDE-NEXT: store i32 [[N]], ptr [[P]], align 4
; WIDE-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1
; WIDE-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]]
-; WIDE-NEXT: br i1 [[C]], label [[LOOP_1:%.*]], label [[EXIT:%.*]]
-; WIDE: loop.1:
+; WIDE-NEXT: br i1 [[C]], label %[[LOOP_1:.*]], label %[[EXIT:.*]]
+; WIDE: [[LOOP_1]]:
; WIDE-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4
; WIDE-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2
; WIDE-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4
@@ -190,9 +196,9 @@ define i32 @max_or_zero(i32 %n, ptr %p) {
; WIDE-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4
; WIDE-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5
; WIDE-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4
-; WIDE-NEXT: br label [[EXIT]]
-; WIDE: exit:
-; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], [[LOOP]] ], [ [[I_NEXT_4]], [[LOOP_1]] ]
+; WIDE-NEXT: br label %[[EXIT]]
+; WIDE: [[EXIT]]:
+; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], %[[LOOP]] ], [ [[I_NEXT_4]], %[[LOOP_1]] ]
; WIDE-NEXT: ret i32 [[I_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index f4a94f552bd57..afaf0e04c41aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -15,8 +15,6 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -66,7 +64,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP31]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP2]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 6a78f2b2bcf2b..75fa3b9b665ae 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -989,8 +989,6 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NOTF-NEXT: [[TMP47:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP47]]
-; CHECK-NOTF-NEXT: [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT: [[TMP51:%.*]] = shl nuw i64 [[TMP48]], 2
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -1078,7 +1076,7 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP51]]
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP47]]
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index 0d195f006cc81..f12f9f79080b6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -222,8 +222,6 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-2-VLA-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; INTERLEAVE-2-VLA-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-2-VLA-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; INTERLEAVE-2-VLA: vector.memcheck:
; INTERLEAVE-2-VLA-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
@@ -264,7 +262,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.iter.check:
-; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
+; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.ph:
; INTERLEAVE-2-VLA-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 33f6ce0766e96..a19af697daa58 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -505,8 +505,6 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -555,7 +553,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP2]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
@@ -607,8 +605,6 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
-; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -657,7 +653,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
+; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 1b46ee158117c..307c68ca14d25 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -12,8 +12,6 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP10]], 2
; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP13]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP6]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; DEFAULT: vector.main.loop.iter.check:
; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 5
@@ -38,7 +36,7 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; DEFAULT: vec.epilog.iter.check:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP15]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP13]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: vec.epilog.ph:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
index 5d9e518fa238d..aaf9c5a540372 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
@@ -80,8 +80,6 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vector.main.loop.iter.check:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 5
@@ -106,7 +104,7 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP11]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.ph:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 50942fc5adafb..11b1707179c18 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -11,8 +11,6 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
@@ -37,7 +35,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index e0a1b2bb6a03e..eabd2557ecd11 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -241,8 +241,6 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP2]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 4
@@ -267,7 +265,7 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]]
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP7]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP9]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
index c7b1bcdf31e45..e7039278977a7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -225,8 +225,6 @@ define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst)
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[UMAX:%.*]] = shl nuw i64 [[TMP2]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
-; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP20]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64
@@ -283,7 +281,7 @@ define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst)
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP29]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
index a16eafda340a4..d61e4d2799e27 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
@@ -13,8 +13,6 @@ define void @test(ptr noalias %A, i64 %v, i64 %n) {
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH1:.*]], label %[[VECTOR_PH1:.*]]
; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP0]], 2
@@ -42,7 +40,7 @@ define void @test(ptr noalias %A, i64 %v, i64 %n) {
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N1]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_PH1]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH1]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
index cf06100f988ef..8680bf8fa1a40 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -147,8 +147,6 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP12]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP11]], 5
@@ -204,7 +202,7 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP14]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP12]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
>From c6fcba83df9a578fc416aab114454c51728f3312 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Thu, 6 Aug 2026 09:33:56 +0000
Subject: [PATCH 5/5] Rebase on top of 208500
---
.../LoopVectorize/AArch64/induction-costs-sve.ll | 4 +++-
.../LoopVectorize/AArch64/interleave-with-gaps.ll | 4 +++-
.../LoopVectorize/AArch64/interleaving-load-store.ll | 4 +++-
.../AArch64/reduction-recurrence-costs-sve.ll | 8 ++++++--
.../Transforms/LoopVectorize/AArch64/store-costs-sve.ll | 4 +++-
.../LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll | 4 +++-
.../LoopVectorize/AArch64/sve2-histcnt-epilogue.ll | 4 +++-
.../test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll | 4 +++-
...ransform-narrow-interleave-to-widen-memory-scalable.ll | 4 +++-
.../transform-narrow-interleave-vscale-x-UF-step.ll | 4 +++-
.../Transforms/LoopVectorize/AArch64/vector-reverse.ll | 4 +++-
11 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index afaf0e04c41aa..f4a94f552bd57 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -15,6 +15,8 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; DEFAULT-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP31:%.*]] = shl nuw i64 [[TMP7]], 3
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
@@ -64,7 +66,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP2]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP31]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 75fa3b9b665ae..6a78f2b2bcf2b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -989,6 +989,8 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NOTF-NEXT: [[TMP47:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP47]]
+; CHECK-NOTF-NEXT: [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP51:%.*]] = shl nuw i64 [[TMP48]], 2
; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-NOTF: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -1076,7 +1078,7 @@ define i32 @load_factor_4_with_gap_reverse(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-NOTF: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP47]]
+; CHECK-NOTF-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP51]]
; CHECK-NOTF-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF10]]
; CHECK-NOTF: [[SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
index f12f9f79080b6..0d195f006cc81 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
@@ -222,6 +222,8 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-2-VLA-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; INTERLEAVE-2-VLA-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
+; INTERLEAVE-2-VLA-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; INTERLEAVE-2-VLA-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; INTERLEAVE-2-VLA: vector.memcheck:
; INTERLEAVE-2-VLA-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
@@ -262,7 +264,7 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
; INTERLEAVE-2-VLA-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; INTERLEAVE-2-VLA: vec.epilog.iter.check:
-; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
+; INTERLEAVE-2-VLA-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
; INTERLEAVE-2-VLA-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; INTERLEAVE-2-VLA: vec.epilog.ph:
; INTERLEAVE-2-VLA-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index a19af697daa58..33f6ce0766e96 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -505,6 +505,8 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -553,7 +555,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP2]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP6]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
@@ -605,6 +607,8 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; VSCALEFORTUNING2-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP1]], 2
; VSCALEFORTUNING2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], [[TMP5]]
+; VSCALEFORTUNING2-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; VSCALEFORTUNING2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; VSCALEFORTUNING2-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
@@ -653,7 +657,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
; VSCALEFORTUNING2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; VSCALEFORTUNING2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; VSCALEFORTUNING2: [[VEC_EPILOG_ITER_CHECK]]:
-; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
+; VSCALEFORTUNING2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP9]]
; VSCALEFORTUNING2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; VSCALEFORTUNING2: [[VEC_EPILOG_PH]]:
; VSCALEFORTUNING2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 307c68ca14d25..1b46ee158117c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -12,6 +12,8 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP10]], 2
; DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP13]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP15:%.*]] = shl nuw i64 [[TMP6]], 2
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; DEFAULT: vector.main.loop.iter.check:
; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 5
@@ -36,7 +38,7 @@ define void @cost_store_i8(ptr %dst) #0 {
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; DEFAULT: vec.epilog.iter.check:
-; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP13]]
+; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP15]]
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: vec.epilog.ph:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
index aaf9c5a540372..5d9e518fa238d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
@@ -80,6 +80,8 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP8]], 4
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vector.main.loop.iter.check:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 5
@@ -104,7 +106,7 @@ define void @main_vf_vscale_x_16(ptr %A, i64 %n) #0 {
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.iter.check:
-; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
+; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP11]]
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK-EPILOG-PREFER-SCALABLE: vec.epilog.ph:
; CHECK-EPILOG-PREFER-SCALABLE-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 11b1707179c18..50942fc5adafb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -11,6 +11,8 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
@@ -35,7 +37,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP5]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index eabd2557ecd11..e0a1b2bb6a03e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -241,6 +241,8 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP2]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 4
@@ -265,7 +267,7 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]]
; CHECK-NEXT: br i1 [[CMP_N1]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP9]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC]], [[TMP7]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ENTRY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
index e7039278977a7..c7b1bcdf31e45 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -225,6 +225,8 @@ define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst)
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[UMAX:%.*]] = shl nuw i64 [[TMP2]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP20]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64
@@ -281,7 +283,7 @@ define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst)
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[UMAX]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP29]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
index d61e4d2799e27..a16eafda340a4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
@@ -13,6 +13,8 @@ define void @test(ptr noalias %A, i64 %v, i64 %n) {
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH1:.*]], label %[[VECTOR_PH1:.*]]
; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP0]], 2
@@ -40,7 +42,7 @@ define void @test(ptr noalias %A, i64 %v, i64 %n) {
; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N1]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP1]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_PH1]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH1]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
index 8680bf8fa1a40..cf06100f988ef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll
@@ -147,6 +147,8 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP12]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP3]], 3
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP11]], 5
@@ -202,7 +204,7 @@ define i32 @reverse_store_with_partial_reduction(ptr noalias %dst, ptr noalias %
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP12]]
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP14]]
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
More information about the llvm-commits
mailing list