[llvm] [LoopUnroll] Relax fixed non-latch exit probability threshold for runtime multi-exit loop unrolling (PR #224325)
Marek Sedláček via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 02:59:23 PDT 2026
https://github.com/mark-sed updated https://github.com/llvm/llvm-project/pull/224325
>From 1865c8519f2b770b62545355891eaae6d72d04eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Sedl=C3=A1=C4=8Dek?= <mr.mareksedlacek at gmail.com>
Date: Thu, 17 Sep 2026 14:33:34 +0000
Subject: [PATCH 1/2] [LoopUnroll] Allow unrolling of multi-exit loops with
small trip count Currently small linear scan loops don't get unrolled because
of the 1/100 threshold we have for unrolling. Loops with trip count < 100
will always have probability bigger than 1/100 threshold. This patch adds a
trip count check to allow unrolling of such cases.
---
.../Transforms/Utils/LoopUnrollRuntime.cpp | 19 +-
...unroll-multi-exit-loop-small-trip-count.ll | 169 ++++++++++++++++++
2 files changed, 185 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 92776cd1472bf..1dc2809e3d92d 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -54,6 +54,11 @@ static cl::opt<bool> UnrollRuntimeMultiExit(
static cl::opt<bool> UnrollRuntimeOtherExitPredictable(
"unroll-runtime-other-exit-predictable", cl::init(false), cl::Hidden,
cl::desc("Assume the non latch exit block to be predictable"));
+static cl::opt<unsigned> UnrollRuntimeMultiExitMinBodies(
+ "unroll-runtime-multi-exit-min-bodies", cl::init(1), cl::Hidden,
+ cl::desc("Minimum number of complete unrolled bodies a multi-exit loop is "
+ "expected to run before taking a non-latch exit, for runtime "
+ "unrolling to kick in"));
// Probability that the loop trip count is so small that after the prolog
// we do not enter the unrolled loop at all.
@@ -535,11 +540,11 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
return NewLoop;
}
-/// Returns true if we can profitably unroll the multi-exit loop L.
+/// Returns true if we can profitably unroll the multi-exit loop L by \p Count.
static bool canProfitablyRuntimeUnrollMultiExitLoop(
Loop *L, const TargetTransformInfo *TTI,
SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
- bool UseEpilogRemainder) {
+ bool UseEpilogRemainder, unsigned Count) {
// The main pain point with multi-exit loop unrolling is that once unrolled,
// we will not be able to merge all blocks into a straight line code.
@@ -585,6 +590,14 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
// If BranchProbability could not be extracted (returns unknown), then
// don't return and do the check for deopt block.
if (!BranchProb.isUnknown()) {
+ // Special case for linear scan loop with trip count < ~100, which will
+ // not pass the P < 1/100 check bellow.
+ if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
+ uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
+ return BranchProb <=
+ BranchProbability::getBranchProbability(1, MinIterations);
+ }
+
auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
return BranchProb < Threshold;
}
@@ -734,7 +747,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
// it is profitable or the general profitability heuristics apply.
if (!RuntimeUnrollMultiExit &&
!canProfitablyRuntimeUnrollMultiExitLoop(
- L, TTI, OtherExits, LatchExit, UseEpilogRemainder)) {
+ L, TTI, OtherExits, LatchExit, UseEpilogRemainder, Count)) {
LLVM_DEBUG(dbgs() << "Multiple exit/exiting blocks in loop and "
"multi-exit unrolling not enabled!\n");
return false;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
new file mode 100644
index 0000000000000..1510fcb520472
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
@@ -0,0 +1,169 @@
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -predictable-branch-threshold=99 -S | FileCheck %s
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES4
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES0
+
+; Every function below is the same backward linear scan.
+
+; size=200: expected scan of ~96 elements, P = 1/97 = 0.0103.
+;
+; This is the case the change was written for. It misses the fixed 1/100
+; threshold by 3% and so was not unrolled, even though ~97 iterations leave
+; room for an 8x body.
+define i32 @scan_p97_size_200(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p97_size_200(
+; CHECK: header:
+; CHECK-COUNT-8: icmp eq i32 %{{.*}}, %key
+; CHECK: header.epil:
+; CHECK: icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES4-LABEL: @scan_p97_size_200(
+; MINBODIES4: header:
+; MINBODIES4-COUNT-8: icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES0-LABEL: @scan_p97_size_200(
+; MINBODIES0-NOT: epil
+;
+entry:
+ br label %header
+
+header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+ %v = load i32, ptr %arrayidx, align 4
+ %match = icmp eq i32 %v, %key
+ br i1 %match, label %found, label %latch, !prof !0
+
+latch:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %notfound, label %header
+
+notfound:
+ ret i32 -1
+
+found:
+ %idx = trunc i64 %i to i32
+ ret i32 %idx
+}
+
+!0 = !{!"branch_weights", i32 1, i32 96}
+
+; size=20: expected scan of ~10 elements, P = 1/11 = 0.0909.
+;
+; The marginal case. 1/P = 11 clears the default requirement of 8 but only just,
+; so the unrolled body runs about once before control falls into the epilogue.
+; Requiring four bodies (1/32) rejects it.
+define i32 @scan_p11_size_20(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p11_size_20(
+; CHECK: header:
+; CHECK-COUNT-8: icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES4-LABEL: @scan_p11_size_20(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p11_size_20(
+; MINBODIES0-NOT: epil
+;
+entry:
+ br label %header
+
+header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+ %v = load i32, ptr %arrayidx, align 4
+ %match = icmp eq i32 %v, %key
+ br i1 %match, label %found, label %latch, !prof !1
+
+latch:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %notfound, label %header
+
+notfound:
+ ret i32 -1
+
+found:
+ %idx = trunc i64 %i to i32
+ ret i32 %idx
+}
+
+!1 = !{!"branch_weights", i32 1, i32 10}
+
+; size=10: expected scan of ~5 elements, P = 1/6 = 0.167.
+;
+; Deliberately still rejected, at every setting.
+define i32 @scan_p6_size_10(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p6_size_10(
+; CHECK-NOT: epil
+;
+; MINBODIES4-LABEL: @scan_p6_size_10(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p6_size_10(
+; MINBODIES0-NOT: epil
+;
+entry:
+ br label %header
+
+header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+ %v = load i32, ptr %arrayidx, align 4
+ %match = icmp eq i32 %v, %key
+ br i1 %match, label %found, label %latch, !prof !2
+
+latch:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %notfound, label %header
+
+notfound:
+ ret i32 -1
+
+found:
+ %idx = trunc i64 %i to i32
+ ret i32 %idx
+}
+
+!2 = !{!"branch_weights", i32 1, i32 5}
+
+; size=20 scan again, but forced to unroll by 16 instead of 8.
+;
+; Identical profile to @scan_p11_size_20, opposite decision. The requirement
+; is now 1/P >= 16 and 1/P = 11. This is the property a fixed probability
+; threshold cannot express at all.
+define i32 @scan_p11_unroll_16(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p11_unroll_16(
+; CHECK-NOT: epil
+;
+; MINBODIES4-LABEL: @scan_p11_unroll_16(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p11_unroll_16(
+; MINBODIES0-NOT: epil
+;
+entry:
+ br label %header
+
+header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+ %v = load i32, ptr %arrayidx, align 4
+ %match = icmp eq i32 %v, %key
+ br i1 %match, label %found, label %latch, !prof !1
+
+latch:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %notfound, label %header, !llvm.loop !3
+
+notfound:
+ ret i32 -1
+
+found:
+ %idx = trunc i64 %i to i32
+ ret i32 %idx
+}
+
+!3 = distinct !{!3, !4}
+!4 = !{!"llvm.loop.unroll.count", i32 16}
>From 9f08e3568aedda9b25c01fa1004f63a6f5dd9e93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Sedl=C3=A1=C4=8Dek?= <mr.mareksedlacek at gmail.com>
Date: Wed, 30 Sep 2026 09:58:47 +0000
Subject: [PATCH 2/2] Removed old heuristic
---
.../Transforms/Utils/LoopUnrollRuntime.cpp | 28 ++++++++-----------
...unroll-multi-exit-loop-small-trip-count.ll | 6 ++--
2 files changed, 14 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 1dc2809e3d92d..1896caec9021e 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -542,8 +542,7 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
/// Returns true if we can profitably unroll the multi-exit loop L by \p Count.
static bool canProfitablyRuntimeUnrollMultiExitLoop(
- Loop *L, const TargetTransformInfo *TTI,
- SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
+ Loop *L, SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
bool UseEpilogRemainder, unsigned Count) {
// The main pain point with multi-exit loop unrolling is that once unrolled,
@@ -578,9 +577,11 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
if (UnrollRuntimeOtherExitPredictable)
return true;
- // The second heuristic is that L has one exit other than the latchexit and
- // that exit is highly unlikely.
- if (TTI) {
+ // Require the loop to run at least MinBodies complete unrolled bodies before
+ // leaving through the non-latch exit.
+ // If UnrollRuntimeMultiExitMinBodies is set to 0, then fall through to the
+ // deopt block check.
+ if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
BasicBlock *LatchBB = L->getLoopLatch();
assert(LatchBB && "Expected loop to have a latch");
BasicBlock *NonLatchExitingBlock =
@@ -590,16 +591,9 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
// If BranchProbability could not be extracted (returns unknown), then
// don't return and do the check for deopt block.
if (!BranchProb.isUnknown()) {
- // Special case for linear scan loop with trip count < ~100, which will
- // not pass the P < 1/100 check bellow.
- if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
- uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
- return BranchProb <=
- BranchProbability::getBranchProbability(1, MinIterations);
- }
-
- auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
- return BranchProb < Threshold;
+ uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
+ return BranchProb <=
+ BranchProbability::getBranchProbability(1, MinIterations);
}
}
@@ -746,8 +740,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
// Otherwise perform multi-exit unrolling, if either the target indicates
// it is profitable or the general profitability heuristics apply.
if (!RuntimeUnrollMultiExit &&
- !canProfitablyRuntimeUnrollMultiExitLoop(
- L, TTI, OtherExits, LatchExit, UseEpilogRemainder, Count)) {
+ !canProfitablyRuntimeUnrollMultiExitLoop(L, OtherExits, LatchExit,
+ UseEpilogRemainder, Count)) {
LLVM_DEBUG(dbgs() << "Multiple exit/exiting blocks in loop and "
"multi-exit unrolling not enabled!\n");
return false;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
index 1510fcb520472..c58cf5aa793d7 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
@@ -1,6 +1,6 @@
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -predictable-branch-threshold=99 -S | FileCheck %s
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES4
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES0
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -S | FileCheck %s
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -S | FileCheck %s -check-prefix=MINBODIES4
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -S | FileCheck %s -check-prefix=MINBODIES0
; Every function below is the same backward linear scan.
More information about the llvm-commits
mailing list