[llvm] [LoopUnroll] Add UnrollMultiExit to loop unrolling options (NFC) (PR #124462)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jan 26 13:42:52 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/124462
>From 59944633753d096e6c8418561f568f90788a6112 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 26 Jan 2025 12:46:23 +0000
Subject: [PATCH 1/3] [LoopUnroll] Add UnrollMultiExit to loop unrolling
options (NFC)
Add an extra knob to UnrollingPreferences to let backends control
whether to allow multi-exit unrolling on a per-loop basis.
This gives backends more fine-grained control on deciding if multi-exit
unrolling is profitable for a given loop and uarch. Similar to
4226e0a0c75.
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 3 +++
llvm/include/llvm/Transforms/Utils/UnrollLoop.h | 4 +++-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 2 ++
llvm/lib/Transforms/Utils/LoopUnroll.cpp | 9 +++++----
llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp | 6 +++---
llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp | 2 +-
6 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 71b204f9c3fec7..c8a34bdc801085 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -622,6 +622,9 @@ class TargetTransformInfo {
/// Don't allow runtime unrolling if expanding the trip count takes more
/// than SCEVExpansionBudget.
unsigned SCEVExpansionBudget;
+ /// Allow runtime unrolling multi-exit loops. Should only be set if the
+ /// target determined that multi-exit unrolling is profitable for the loop.
+ bool UnrollMultiExit;
};
/// Get target-customized preferences for the generic loop unrolling
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 8cf17ced458c82..3f9c14880ad471 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -76,6 +76,7 @@ struct UnrollLoopOptions {
bool ForgetAllSCEV;
const Instruction *Heart = nullptr;
unsigned SCEVExpansionBudget;
+ bool UnrollMultiExit = false;
};
LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -91,7 +92,8 @@ bool UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop = nullptr);
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit,
+ Loop **ResultLoop = nullptr);
LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 090348809e5719..84e4630640134e 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -220,6 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.UnrollAndJamInnerLoopThreshold = 60;
UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
+ UP.UnrollMultiExit = false;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1352,6 +1353,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ULO.ForgetAllSCEV = ForgetAllSCEV;
ULO.Heart = getLoopConvergenceHeart(L);
ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
+ ULO.UnrollMultiExit = UP.UnrollMultiExit;
LoopUnrollResult UnrollResult = UnrollLoop(
L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index b11d92836a998f..623031f532ab32 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -590,10 +590,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
: isEpilogProfitable(L);
if (ULO.Runtime &&
- !UnrollRuntimeLoopRemainder(
- L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
- ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
- PreserveLCSSA, ULO.SCEVExpansionBudget, RemainderLoop)) {
+ !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
+ EpilogProfitability, ULO.UnrollRemainder,
+ ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
+ PreserveLCSSA, ULO.SCEVExpansionBudget,
+ ULO.UnrollMultiExit, RemainderLoop)) {
if (ULO.Force)
ULO.Runtime = false;
else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index b0bc55cd64c377..63210c75d697a7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -583,7 +583,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop) {
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit, Loop **ResultLoop) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,8 +632,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
if (!PreserveLCSSA)
return false;
- if (!canProfitablyUnrollMultiExitLoop(L, OtherExits, LatchExit,
- UseEpilogRemainder)) {
+ if (!UnrollMultiExit && !canProfitablyUnrollMultiExitLoop(
+ L, OtherExits, LatchExit, UseEpilogRemainder)) {
LLVM_DEBUG(
dbgs()
<< "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index b49e37d9eee984..fb02c89c77a10e 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -73,6 +73,6 @@ while.end: ; preds = %while.cond
bool ret =
UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, &SE, &DT,
- &AC, /*TTI=*/nullptr, PreserveLCSSA, 4);
+ &AC, /*TTI=*/nullptr, PreserveLCSSA, 4, false);
EXPECT_FALSE(ret);
}
>From e2ae7f879ee005bd0d878a452c3194f917d8fcfe Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 26 Jan 2025 21:34:40 +0000
Subject: [PATCH 2/3] !fixup Rename to RuntimeUnrolMultiExit
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
llvm/include/llvm/Transforms/Utils/UnrollLoop.h | 4 ++--
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 4 ++--
llvm/lib/Transforms/Utils/LoopUnroll.cpp | 2 +-
llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp | 6 +++---
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index c8a34bdc801085..51256271a026f6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -624,7 +624,7 @@ class TargetTransformInfo {
unsigned SCEVExpansionBudget;
/// Allow runtime unrolling multi-exit loops. Should only be set if the
/// target determined that multi-exit unrolling is profitable for the loop.
- bool UnrollMultiExit;
+ bool RuntimeUnrollMultiExit;
};
/// Get target-customized preferences for the generic loop unrolling
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 3f9c14880ad471..ed560f6f6e2fa9 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -76,7 +76,7 @@ struct UnrollLoopOptions {
bool ForgetAllSCEV;
const Instruction *Heart = nullptr;
unsigned SCEVExpansionBudget;
- bool UnrollMultiExit = false;
+ bool RuntimeUnrollMultiExit = false;
};
LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -92,7 +92,7 @@ bool UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, bool UnrollMultiExit,
+ unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
Loop **ResultLoop = nullptr);
LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 84e4630640134e..5f28ee616f0e4f 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -220,7 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.UnrollAndJamInnerLoopThreshold = 60;
UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
- UP.UnrollMultiExit = false;
+ UP.RuntimeUnrollMultiExit = false;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1353,7 +1353,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ULO.ForgetAllSCEV = ForgetAllSCEV;
ULO.Heart = getLoopConvergenceHeart(L);
ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
- ULO.UnrollMultiExit = UP.UnrollMultiExit;
+ ULO.RuntimeUnrollMultiExit = UP.RuntimeUnrollMultiExit;
LoopUnrollResult UnrollResult = UnrollLoop(
L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 623031f532ab32..af6f9c2a8edd3f 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -594,7 +594,7 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
EpilogProfitability, ULO.UnrollRemainder,
ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
PreserveLCSSA, ULO.SCEVExpansionBudget,
- ULO.UnrollMultiExit, RemainderLoop)) {
+ ULO.RuntimeUnrollMultiExit, RemainderLoop)) {
if (ULO.Force)
ULO.Runtime = false;
else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 63210c75d697a7..11c455ea836c63 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -461,7 +461,7 @@ CloneLoopBlocks(Loop *L, Value *NewIter, const bool UseEpilogRemainder,
/// Returns true if we can profitably unroll the multi-exit loop L. Currently,
/// we return true only if UnrollRuntimeMultiExit is set to true.
-static bool canProfitablyUnrollMultiExitLoop(
+static bool canProfitablyRuntimeUnrollMultiExitLoop(
Loop *L, SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
bool UseEpilogRemainder) {
@@ -583,7 +583,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, bool UnrollMultiExit, Loop **ResultLoop) {
+ unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit, Loop **ResultLoop) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,7 +632,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
if (!PreserveLCSSA)
return false;
- if (!UnrollMultiExit && !canProfitablyUnrollMultiExitLoop(
+ if (!RuntimeUnrollMultiExit && !canProfitablyRuntimeUnrollMultiExitLoop(
L, OtherExits, LatchExit, UseEpilogRemainder)) {
LLVM_DEBUG(
dbgs()
>From f1b43d3018f8e233b2bed7c17724ac20557fa474 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 26 Jan 2025 21:42:14 +0000
Subject: [PATCH 3/3] !fixup fix formatting
---
llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 11c455ea836c63..524b268aee2f36 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -583,7 +583,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit, Loop **ResultLoop) {
+ unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
+ Loop **ResultLoop) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,8 +633,9 @@ bool llvm::UnrollRuntimeLoopRemainder(
if (!PreserveLCSSA)
return false;
- if (!RuntimeUnrollMultiExit && !canProfitablyRuntimeUnrollMultiExitLoop(
- L, OtherExits, LatchExit, UseEpilogRemainder)) {
+ if (!RuntimeUnrollMultiExit &&
+ !canProfitablyRuntimeUnrollMultiExitLoop(L, OtherExits, LatchExit,
+ UseEpilogRemainder)) {
LLVM_DEBUG(
dbgs()
<< "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
More information about the llvm-commits
mailing list