[llvm] [IRCE] Relax profitability check (PR #104659)
Jan Ječmen via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 21 08:20:29 PDT 2024
https://github.com/JanJecmen updated https://github.com/llvm/llvm-project/pull/104659
>From 629c154015721b6dc59b0288583ffa5dcea17a63 Mon Sep 17 00:00:00 2001
From: Jan Jecmen <jjecmen at azul.com>
Date: Sat, 17 Aug 2024 01:19:46 +0000
Subject: [PATCH 1/3] [IRCE] Relax the profitability check
IRCE currently has two profitability checks:
1/ min number of iterations (10 by default)
2/ branch is highly biased (> 15/16)
However, it may still be profitable to eliminate range checks even if the branch isn't as biased. Consider, for example, a loop with 100 iterations, where IRCE currently eliminates all 100 range checks. The same range checks performed over a loop with 200 iterations aren't eliminated because the branch is 50-50.
This patch proposes to relax the profitability checks of IRCE. Namely, instead of the two checks currenly in place, consider IRCE profitable if the branch probability scaled by the expected number of iterations (i.e., the estimated number of eliminated checks) is over a threshold. This covers the minimum number of iterations check (there are at least as many iterations as eliminated range checks), and changes the bias check from a percent of iterations to at least a constant threshold of eliminated checks.
If the number of iterations can't be estimated, the check falls back to the current 15/16 likelihood check.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 100 +++++++++---------
1 file changed, 48 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 104e8ceb796700..96850e11f1af24 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -107,8 +107,8 @@ static cl::opt<bool> PrintRangeChecks("irce-print-range-checks", cl::Hidden,
static cl::opt<bool> SkipProfitabilityChecks("irce-skip-profitability-checks",
cl::Hidden, cl::init(false));
-static cl::opt<unsigned> MinRuntimeIterations("irce-min-runtime-iterations",
- cl::Hidden, cl::init(10));
+static cl::opt<unsigned> MinEliminatedChecks("irce-min-eliminated-checks",
+ cl::Hidden, cl::init(10));
static cl::opt<bool> AllowUnsignedLatchCondition("irce-allow-unsigned-latch",
cl::Hidden, cl::init(true));
@@ -132,15 +132,9 @@ static cl::opt<bool>
namespace {
-/// An inductive range check is conditional branch in a loop with
-///
-/// 1. a very cold successor (i.e. the branch jumps to that successor very
-/// rarely)
-///
-/// and
-///
-/// 2. a condition that is provably true for some contiguous range of values
-/// taken by the containing loop's induction variable.
+/// An inductive range check is conditional branch in a loop with a condition
+/// that is provably true for some contiguous range of values taken by the
+/// containing loop's induction variable.
///
class InductiveRangeCheck {
@@ -235,6 +229,7 @@ class InductiveRangeCheck {
/// checks, and hence don't end up in \p Checks.
static void extractRangeChecksFromBranch(
BranchInst *BI, Loop *L, ScalarEvolution &SE, BranchProbabilityInfo *BPI,
+ std::optional<unsigned int> EstimatedTripCount,
SmallVectorImpl<InductiveRangeCheck> &Checks, bool &Changed);
};
@@ -248,9 +243,11 @@ class InductiveRangeCheckElimination {
std::optional<llvm::function_ref<llvm::BlockFrequencyInfo &()>>;
GetBFIFunc GetBFI;
- // Returns true if it is profitable to do a transform basing on estimation of
- // number of iterations.
- bool isProfitableToTransform(const Loop &L, LoopStructure &LS);
+ // Returns the estimated number of iterations based on block frequency info if
+ // available, or on branch probability info. Nullopt is returned if the number
+ // of iterations cannot be estimated.
+ std::optional<unsigned int> estimatedTripCount(const Loop &L,
+ LoopStructure &LS);
public:
InductiveRangeCheckElimination(ScalarEvolution &SE,
@@ -524,6 +521,7 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
void InductiveRangeCheck::extractRangeChecksFromBranch(
BranchInst *BI, Loop *L, ScalarEvolution &SE, BranchProbabilityInfo *BPI,
+ std::optional<unsigned int> EstimatedTripCount,
SmallVectorImpl<InductiveRangeCheck> &Checks, bool &Changed) {
if (BI->isUnconditional() || BI->getParent() == L->getLoopLatch())
return;
@@ -531,11 +529,19 @@ void InductiveRangeCheck::extractRangeChecksFromBranch(
unsigned IndexLoopSucc = L->contains(BI->getSuccessor(0)) ? 0 : 1;
assert(L->contains(BI->getSuccessor(IndexLoopSucc)) &&
"No edges coming to loop?");
- BranchProbability LikelyTaken(15, 16);
- if (!SkipProfitabilityChecks && BPI &&
- BPI->getEdgeProbability(BI->getParent(), IndexLoopSucc) < LikelyTaken)
- return;
+ if (!SkipProfitabilityChecks && BPI) {
+ auto SuccessProbability =
+ BPI->getEdgeProbability(BI->getParent(), IndexLoopSucc);
+ if (EstimatedTripCount) {
+ if (SuccessProbability.scale(*EstimatedTripCount) < MinEliminatedChecks)
+ return;
+ } else {
+ BranchProbability LikelyTaken(15, 16);
+ if (SuccessProbability < LikelyTaken)
+ return;
+ }
+ }
// IRCE expects branch's true edge comes to loop. Invert branch for opposite
// case.
@@ -940,35 +946,25 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
return getLoopPassPreservedAnalyses();
}
-bool
-InductiveRangeCheckElimination::isProfitableToTransform(const Loop &L,
- LoopStructure &LS) {
- if (SkipProfitabilityChecks)
- return true;
+std::optional<unsigned int>
+InductiveRangeCheckElimination::estimatedTripCount(const Loop &L,
+ LoopStructure &LS) {
if (GetBFI) {
BlockFrequencyInfo &BFI = (*GetBFI)();
uint64_t hFreq = BFI.getBlockFreq(LS.Header).getFrequency();
uint64_t phFreq = BFI.getBlockFreq(L.getLoopPreheader()).getFrequency();
- if (phFreq != 0 && hFreq != 0 && (hFreq / phFreq < MinRuntimeIterations)) {
- LLVM_DEBUG(dbgs() << "irce: could not prove profitability: "
- << "the estimated number of iterations basing on "
- "frequency info is " << (hFreq / phFreq) << "\n";);
- return false;
- }
- return true;
+ if (phFreq == 0 || hFreq == 0)
+ return std::nullopt;
+ return {hFreq / phFreq};
}
if (!BPI)
- return true;
+ return std::nullopt;
BranchProbability ExitProbability =
BPI->getEdgeProbability(LS.Latch, LS.LatchBrExitIdx);
- if (ExitProbability > BranchProbability(1, MinRuntimeIterations)) {
- LLVM_DEBUG(dbgs() << "irce: could not prove profitability: "
- << "the exit probability is too big " << ExitProbability
- << "\n";);
- return false;
- }
- return true;
+ if (ExitProbability.isUnknown() || ExitProbability.isZero())
+ return std::nullopt;
+ return {ExitProbability.getDenominator() / ExitProbability.getNumerator()};
}
bool InductiveRangeCheckElimination::run(
@@ -985,13 +981,25 @@ bool InductiveRangeCheckElimination::run(
}
LLVMContext &Context = Preheader->getContext();
+
+ const char *FailureReason = nullptr;
+ std::optional<LoopStructure> MaybeLoopStructure =
+ LoopStructure::parseLoopStructure(SE, *L, AllowUnsignedLatchCondition,
+ FailureReason);
+ if (!MaybeLoopStructure) {
+ LLVM_DEBUG(dbgs() << "irce: could not parse loop structure: "
+ << FailureReason << "\n";);
+ return false;
+ }
+ LoopStructure LS = *MaybeLoopStructure;
+
SmallVector<InductiveRangeCheck, 16> RangeChecks;
bool Changed = false;
for (auto *BBI : L->getBlocks())
if (BranchInst *TBI = dyn_cast<BranchInst>(BBI->getTerminator()))
- InductiveRangeCheck::extractRangeChecksFromBranch(TBI, L, SE, BPI,
- RangeChecks, Changed);
+ InductiveRangeCheck::extractRangeChecksFromBranch(
+ TBI, L, SE, BPI, estimatedTripCount(*L, LS), RangeChecks, Changed);
if (RangeChecks.empty())
return Changed;
@@ -1009,18 +1017,6 @@ bool InductiveRangeCheckElimination::run(
if (PrintRangeChecks)
PrintRecognizedRangeChecks(errs());
- const char *FailureReason = nullptr;
- std::optional<LoopStructure> MaybeLoopStructure =
- LoopStructure::parseLoopStructure(SE, *L, AllowUnsignedLatchCondition,
- FailureReason);
- if (!MaybeLoopStructure) {
- LLVM_DEBUG(dbgs() << "irce: could not parse loop structure: "
- << FailureReason << "\n";);
- return Changed;
- }
- LoopStructure LS = *MaybeLoopStructure;
- if (!isProfitableToTransform(*L, LS))
- return Changed;
const SCEVAddRecExpr *IndVar =
cast<SCEVAddRecExpr>(SE.getMinusSCEV(SE.getSCEV(LS.IndVarBase), SE.getSCEV(LS.IndVarStep)));
>From 7c093ededc752ae5bf5ad2495e05028384faf747 Mon Sep 17 00:00:00 2001
From: Jan Jecmen <jjecmen at azul.com>
Date: Sat, 17 Aug 2024 01:58:39 +0000
Subject: [PATCH 2/3] Fix test
---
llvm/test/Transforms/IRCE/low-iterations.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/IRCE/low-iterations.ll b/llvm/test/Transforms/IRCE/low-iterations.ll
index 071ab4d0156852..e044c455fe6e2b 100644
--- a/llvm/test/Transforms/IRCE/low-iterations.ll
+++ b/llvm/test/Transforms/IRCE/low-iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-runtime-iterations=3 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO
-; RUN: opt -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-runtime-iterations=0 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-YES
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-eliminated-checks=3 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-eliminated-checks=0 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-YES
; CHECK-YES: constrained Loop
; CHECK-NO-NOT: constrained Loop
>From 6b8311761b58aa2377b13f3d7ada4238562cf26b Mon Sep 17 00:00:00 2001
From: Jan Jecmen <jjecmen at azul.com>
Date: Wed, 21 Aug 2024 15:19:14 +0000
Subject: [PATCH 3/3] Add test and negative test for the new check
---
llvm/test/Transforms/IRCE/profitability.ll | 40 ++++++++++++++++++++++
1 file changed, 40 insertions(+)
create mode 100644 llvm/test/Transforms/IRCE/profitability.ll
diff --git a/llvm/test/Transforms/IRCE/profitability.ll b/llvm/test/Transforms/IRCE/profitability.ll
new file mode 100644
index 00000000000000..abc10ccdd74af6
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/profitability.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-eliminated-checks=11 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO
+; RUN: opt -S -verify-loop-info -irce-print-changed-loops -passes=irce -irce-min-eliminated-checks=10 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-YES
+
+; CHECK-YES: constrained Loop
+; CHECK-NO-NOT: constrained Loop
+
+declare void @bar(i32)
+
+define i32 @foo(
+ ptr %arr_a, ptr %a_len_ptr, i32 %n) {
+
+entry:
+ %len.a = load i32, ptr %a_len_ptr, !range !0
+ %first.itr.check = icmp sgt i32 %n, 0
+ br i1 %first.itr.check, label %loop, label %exit, !prof !1
+
+loop:
+ %idx = phi i32 [ 0, %entry ] , [ %idx.next, %backedge ]
+ %idx.next = add i32 %idx, 1
+ %abc.a = icmp slt i32 %idx, %len.a
+ br i1 %abc.a, label %in.bounds.a, label %backedge, !prof !2
+
+in.bounds.a:
+ %addr.a = getelementptr i32, ptr %arr_a, i32 %idx
+ %val = load i32, ptr %addr.a
+ call void @bar(i32 %val)
+ br label %backedge
+
+backedge:
+ %next = icmp slt i32 %idx.next, %n
+ br i1 %next, label %loop, label %exit, !prof !4
+
+exit:
+ ret i32 0
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 1024, i32 1}
+!2 = !{!"branch_weights", i32 1, i32 9}
+!4 = !{!"branch_weights", i32 99, i32 1}
More information about the llvm-commits
mailing list