[llvm] [LoopUnroll] Relax fixed non-latch exit probability threshold for runtime multi-exit loop unrolling (PR #224325)

Marek Sedláček via llvm-commits llvm-commits at lists.llvm.org
Wed Sep 30 02:59:23 PDT 2026


https://github.com/mark-sed updated https://github.com/llvm/llvm-project/pull/224325

>From 1865c8519f2b770b62545355891eaae6d72d04eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Sedl=C3=A1=C4=8Dek?= <mr.mareksedlacek at gmail.com>
Date: Thu, 17 Sep 2026 14:33:34 +0000
Subject: [PATCH 1/2] [LoopUnroll] Allow unrolling of multi-exit loops with
 small trip count Currently small linear scan loops don't get unrolled because
 of the 1/100 threshold we have for unrolling. Loops with trip count < 100
 will always have probability bigger than 1/100 threshold. This patch adds a
 trip count check to allow unrolling of such cases.

---
 .../Transforms/Utils/LoopUnrollRuntime.cpp    |  19 +-
 ...unroll-multi-exit-loop-small-trip-count.ll | 169 ++++++++++++++++++
 2 files changed, 185 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll

diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 92776cd1472bf..1dc2809e3d92d 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -54,6 +54,11 @@ static cl::opt<bool> UnrollRuntimeMultiExit(
 static cl::opt<bool> UnrollRuntimeOtherExitPredictable(
     "unroll-runtime-other-exit-predictable", cl::init(false), cl::Hidden,
     cl::desc("Assume the non latch exit block to be predictable"));
+static cl::opt<unsigned> UnrollRuntimeMultiExitMinBodies(
+    "unroll-runtime-multi-exit-min-bodies", cl::init(1), cl::Hidden,
+    cl::desc("Minimum number of complete unrolled bodies a multi-exit loop is "
+             "expected to run before taking a non-latch exit, for runtime "
+             "unrolling to kick in"));
 
 // Probability that the loop trip count is so small that after the prolog
 // we do not enter the unrolled loop at all.
@@ -535,11 +540,11 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
   return NewLoop;
 }
 
-/// Returns true if we can profitably unroll the multi-exit loop L.
+/// Returns true if we can profitably unroll the multi-exit loop L by \p Count.
 static bool canProfitablyRuntimeUnrollMultiExitLoop(
     Loop *L, const TargetTransformInfo *TTI,
     SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
-    bool UseEpilogRemainder) {
+    bool UseEpilogRemainder, unsigned Count) {
 
   // The main pain point with multi-exit loop unrolling is that once unrolled,
   // we will not be able to merge all blocks into a straight line code.
@@ -585,6 +590,14 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
     // If BranchProbability could not be extracted (returns unknown), then
     // don't return and do the check for deopt block.
     if (!BranchProb.isUnknown()) {
+      // Special case for linear scan loop with trip count < ~100, which will
+      // not pass the P < 1/100 check bellow.
+      if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
+        uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
+        return BranchProb <=
+               BranchProbability::getBranchProbability(1, MinIterations);
+      }
+
       auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
       return BranchProb < Threshold;
     }
@@ -734,7 +747,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
       // it is profitable or the general profitability heuristics apply.
       if (!RuntimeUnrollMultiExit &&
           !canProfitablyRuntimeUnrollMultiExitLoop(
-              L, TTI, OtherExits, LatchExit, UseEpilogRemainder)) {
+              L, TTI, OtherExits, LatchExit, UseEpilogRemainder, Count)) {
         LLVM_DEBUG(dbgs() << "Multiple exit/exiting blocks in loop and "
                              "multi-exit unrolling not enabled!\n");
         return false;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
new file mode 100644
index 0000000000000..1510fcb520472
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
@@ -0,0 +1,169 @@
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -predictable-branch-threshold=99 -S | FileCheck %s
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES4
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES0
+
+; Every function below is the same backward linear scan.
+
+; size=200: expected scan of ~96 elements, P = 1/97 = 0.0103.
+;
+; This is the case the change was written for. It misses the fixed 1/100
+; threshold by 3% and so was not unrolled, even though ~97 iterations leave
+; room for an 8x body.
+define i32 @scan_p97_size_200(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p97_size_200(
+; CHECK:         header:
+; CHECK-COUNT-8:   icmp eq i32 %{{.*}}, %key
+; CHECK:         header.epil:
+; CHECK:           icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES4-LABEL: @scan_p97_size_200(
+; MINBODIES4:         header:
+; MINBODIES4-COUNT-8:   icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES0-LABEL: @scan_p97_size_200(
+; MINBODIES0-NOT: epil
+;
+entry:
+  br label %header
+
+header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %match = icmp eq i32 %v, %key
+  br i1 %match, label %found, label %latch, !prof !0
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %notfound, label %header
+
+notfound:
+  ret i32 -1
+
+found:
+  %idx = trunc i64 %i to i32
+  ret i32 %idx
+}
+
+!0 = !{!"branch_weights", i32 1, i32 96}
+
+; size=20: expected scan of ~10 elements, P = 1/11 = 0.0909.
+;
+; The marginal case. 1/P = 11 clears the default requirement of 8 but only just,
+; so the unrolled body runs about once before control falls into the epilogue.
+; Requiring four bodies (1/32) rejects it.
+define i32 @scan_p11_size_20(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p11_size_20(
+; CHECK:         header:
+; CHECK-COUNT-8:   icmp eq i32 %{{.*}}, %key
+;
+; MINBODIES4-LABEL: @scan_p11_size_20(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p11_size_20(
+; MINBODIES0-NOT: epil
+;
+entry:
+  br label %header
+
+header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %match = icmp eq i32 %v, %key
+  br i1 %match, label %found, label %latch, !prof !1
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %notfound, label %header
+
+notfound:
+  ret i32 -1
+
+found:
+  %idx = trunc i64 %i to i32
+  ret i32 %idx
+}
+
+!1 = !{!"branch_weights", i32 1, i32 10}
+
+; size=10: expected scan of ~5 elements, P = 1/6 = 0.167.
+;
+; Deliberately still rejected, at every setting.
+define i32 @scan_p6_size_10(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p6_size_10(
+; CHECK-NOT: epil
+;
+; MINBODIES4-LABEL: @scan_p6_size_10(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p6_size_10(
+; MINBODIES0-NOT: epil
+;
+entry:
+  br label %header
+
+header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %match = icmp eq i32 %v, %key
+  br i1 %match, label %found, label %latch, !prof !2
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %notfound, label %header
+
+notfound:
+  ret i32 -1
+
+found:
+  %idx = trunc i64 %i to i32
+  ret i32 %idx
+}
+
+!2 = !{!"branch_weights", i32 1, i32 5}
+
+; size=20 scan again, but forced to unroll by 16 instead of 8.
+;
+; Identical profile to @scan_p11_size_20, opposite decision. The requirement
+; is now 1/P >= 16 and 1/P = 11. This is the property a fixed probability
+; threshold cannot express at all.
+define i32 @scan_p11_unroll_16(ptr nocapture %a, i64 %n, i32 %key) {
+; CHECK-LABEL: @scan_p11_unroll_16(
+; CHECK-NOT: epil
+;
+; MINBODIES4-LABEL: @scan_p11_unroll_16(
+; MINBODIES4-NOT: epil
+;
+; MINBODIES0-LABEL: @scan_p11_unroll_16(
+; MINBODIES0-NOT: epil
+;
+entry:
+  br label %header
+
+header:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %match = icmp eq i32 %v, %key
+  br i1 %match, label %found, label %latch, !prof !1
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %notfound, label %header, !llvm.loop !3
+
+notfound:
+  ret i32 -1
+
+found:
+  %idx = trunc i64 %i to i32
+  ret i32 %idx
+}
+
+!3 = distinct !{!3, !4}
+!4 = !{!"llvm.loop.unroll.count", i32 16}

>From 9f08e3568aedda9b25c01fa1004f63a6f5dd9e93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Sedl=C3=A1=C4=8Dek?= <mr.mareksedlacek at gmail.com>
Date: Wed, 30 Sep 2026 09:58:47 +0000
Subject: [PATCH 2/2] Removed old heuristic

---
 .../Transforms/Utils/LoopUnrollRuntime.cpp    | 28 ++++++++-----------
 ...unroll-multi-exit-loop-small-trip-count.ll |  6 ++--
 2 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 1dc2809e3d92d..1896caec9021e 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -542,8 +542,7 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
 
 /// Returns true if we can profitably unroll the multi-exit loop L by \p Count.
 static bool canProfitablyRuntimeUnrollMultiExitLoop(
-    Loop *L, const TargetTransformInfo *TTI,
-    SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
+    Loop *L, SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
     bool UseEpilogRemainder, unsigned Count) {
 
   // The main pain point with multi-exit loop unrolling is that once unrolled,
@@ -578,9 +577,11 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
   if (UnrollRuntimeOtherExitPredictable)
     return true;
 
-  // The second heuristic is that L has one exit other than the latchexit and
-  // that exit is highly unlikely.
-  if (TTI) {
+  // Require the loop to run at least MinBodies complete unrolled bodies before
+  // leaving through the non-latch exit.
+  // If UnrollRuntimeMultiExitMinBodies is set to 0, then fall through to the
+  // deopt block check.
+  if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
     BasicBlock *LatchBB = L->getLoopLatch();
     assert(LatchBB && "Expected loop to have a latch");
     BasicBlock *NonLatchExitingBlock =
@@ -590,16 +591,9 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
     // If BranchProbability could not be extracted (returns unknown), then
     // don't return and do the check for deopt block.
     if (!BranchProb.isUnknown()) {
-      // Special case for linear scan loop with trip count < ~100, which will
-      // not pass the P < 1/100 check bellow.
-      if (unsigned MinBodies = UnrollRuntimeMultiExitMinBodies) {
-        uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
-        return BranchProb <=
-               BranchProbability::getBranchProbability(1, MinIterations);
-      }
-
-      auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
-      return BranchProb < Threshold;
+      uint64_t MinIterations = static_cast<uint64_t>(MinBodies) * Count;
+      return BranchProb <=
+             BranchProbability::getBranchProbability(1, MinIterations);
     }
   }
 
@@ -746,8 +740,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
       // Otherwise perform multi-exit unrolling, if either the target indicates
       // it is profitable or the general profitability heuristics apply.
       if (!RuntimeUnrollMultiExit &&
-          !canProfitablyRuntimeUnrollMultiExitLoop(
-              L, TTI, OtherExits, LatchExit, UseEpilogRemainder, Count)) {
+          !canProfitablyRuntimeUnrollMultiExitLoop(L, OtherExits, LatchExit,
+                                                   UseEpilogRemainder, Count)) {
         LLVM_DEBUG(dbgs() << "Multiple exit/exiting blocks in loop and "
                              "multi-exit unrolling not enabled!\n");
         return false;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
index 1510fcb520472..c58cf5aa793d7 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-multi-exit-loop-small-trip-count.ll
@@ -1,6 +1,6 @@
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -predictable-branch-threshold=99 -S | FileCheck %s
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES4
-; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -predictable-branch-threshold=99 -S | FileCheck %s -check-prefix=MINBODIES0
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -S | FileCheck %s
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=4 -S | FileCheck %s -check-prefix=MINBODIES4
+; RUN: opt < %s -passes=loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -unroll-runtime-other-exit-predictable=false -unroll-runtime-multi-exit-min-bodies=0 -S | FileCheck %s -check-prefix=MINBODIES0
 
 ; Every function below is the same backward linear scan.
 



More information about the llvm-commits mailing list