[llvm] BranchProbabilityInfo: Report branch_weights 1, 0 for invoke (PR #72947)

Matthias Braun via llvm-commits llvm-commits at lists.llvm.org
Mon Nov 20 19:28:11 PST 2023


https://github.com/MatzeB updated https://github.com/llvm/llvm-project/pull/72947

>From f89ebeb730aba81ba69512064161effed5e93c30 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze at braunis.de>
Date: Mon, 20 Nov 2023 16:56:19 -0800
Subject: [PATCH] BranchProbabilityInfo: Report branch_weights 1, 0 for invoke

The `calcEstimatedHeuristics` used to report probabilities of invoke
exception paths as `BlockExecWeight::UNWIND` in relation to
`BlockExecWeight::DEFAULT`. This is "just" a 0xfffff : 1 ratio which can
lead to overestimated block counts for loops with high execution counts.
For example for a loop with a trip count of 5 million the exception loop
exit was estimated as 4 times more likely than the regular loop exit.

This change circumvents the limited range of `calcEstimatedHeuristics`
by hardcoding the probabilities of exception edge as 1, 0. This also
matches the behavior when loading PGO profiles.
---
 .../llvm/Analysis/BranchProbabilityInfo.h     |  1 +
 llvm/lib/Analysis/BranchProbabilityInfo.cpp   | 16 +++++++++++++++
 .../Analysis/BranchProbabilityInfo/basic.ll   | 20 +++++++++----------
 .../BranchProbabilityInfo/deopt-invoke.ll     | 12 +++++------
 .../Analysis/BranchProbabilityInfo/loop.ll    |  4 ++--
 .../BranchProbabilityInfo/noreturn.ll         |  4 ++--
 6 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index fb02997371bfb87..b0481a55e314d16 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -406,6 +406,7 @@ class BranchProbabilityInfo {
   /// Based on computed weights by \p computeEstimatedBlockWeight set
   /// probabilities on branches.
   bool calcEstimatedHeuristics(const BasicBlock *BB);
+  bool calcFixedWeights(const BasicBlock *BB);
   bool calcMetadataWeights(const BasicBlock *BB);
   bool calcPointerHeuristics(const BasicBlock *BB);
   bool calcZeroHeuristics(const BasicBlock *BB, const TargetLibraryInfo *TLI);
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index b45deccd913db47..4818ced94d5bdc8 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -870,6 +870,20 @@ void BranchProbabilityInfo::computeEestimateBlockWeight(
   } while (!BlockWorkList.empty() || !LoopWorkList.empty());
 }
 
+bool BranchProbabilityInfo::calcFixedWeights(const BasicBlock *BB) {
+  const Instruction *Terminator = BB->getTerminator();
+  if (const InvokeInst *Invoke = dyn_cast<InvokeInst>(Terminator)) {
+    assert(Invoke->getNormalDest() == Invoke->getSuccessor(0) &&
+           Invoke->getUnwindDest() == Invoke->getSuccessor(1) &&
+           "unexpected successor ordering");
+    SmallVector<BranchProbability, 2> EdgeProbs(
+        {BranchProbability::getOne(), BranchProbability::getZero()});
+    setEdgeProbability(BB, EdgeProbs);
+    return true;
+  }
+  return false;
+}
+
 // Calculate edge probabilities based on block's estimated weight.
 // Note that gathered weights were not scaled for loops. Thus edges entering
 // and exiting loops requires special processing.
@@ -1256,6 +1270,8 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI,
       continue;
     if (calcMetadataWeights(BB))
       continue;
+    if (calcFixedWeights(BB))
+      continue;
     if (calcEstimatedHeuristics(BB))
       continue;
     if (calcPointerHeuristics(BB))
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
index 682de24ac5828af..90d34a418d98d47 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/basic.ll
@@ -243,8 +243,8 @@ entry:
 if.then:
   invoke i32 @InvokeCall()
           to label %invoke.cont unwind label %lpad
-; CHECK:  edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
-; CHECK:  edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00%
+; CHECK:  edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK:  edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
 
 invoke.cont:
   call void @ColdFunc() #0
@@ -270,8 +270,8 @@ if.then:
   invoke i32 @InvokeCall()
           to label %invoke.cont unwind label %lpad
 ; The cold call heuristic should not kick in when the cold callsite is in EH path.
-; CHECK: edge if.then -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
-; CHECK: edge if.then -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
+; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
 
 invoke.cont:
   br label %if.end
@@ -298,8 +298,8 @@ if.then:
           to label %invoke.cont unwind label %lpad
 ; Regardless of cold calls, edge weights from a invoke instruction should be
 ; determined by the invoke heuristic.
-; CHECK: edge if.then -> invoke.cont probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
-; CHECK: edge if.then -> lpad probability is 0x00008000 / 0x80000000 = 0.00%
+; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
 
 invoke.cont:
   call void @ColdFunc() #0
@@ -318,13 +318,13 @@ if.end:
 ; CHECK-LABEL: test_invoke_code_profiled
 define void @test_invoke_code_profiled(i1 %c) personality ptr @__gxx_personality_v0 {
 entry:
-; CHECK: edge entry -> invoke.to0 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
-; CHECK: edge entry -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
+; CHECK: edge entry -> invoke.to0 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK: edge entry -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
   invoke i32 @InvokeCall() to label %invoke.to0 unwind label %lpad
 
 invoke.to0:
-; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
-; CHECK: edge invoke.to0 -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
+; CHECK: edge invoke.to0 -> invoke.to1 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK: edge invoke.to0 -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
   invoke i32 @InvokeCall() to label %invoke.to1 unwind label %lpad,
      !prof !{!"branch_weights", i32 444}
 
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll b/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll
index ae46ed7851f895e..1679d0903ef903f 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/deopt-invoke.ll
@@ -11,8 +11,8 @@ declare void @cold() cold
 define void @test1(i32 %0) personality ptr @"personality_function"  !prof !1 {
 ;CHECK: edge entry -> unreached probability is 0x00000001 / 0x80000000 = 0.00%
 ;CHECK: edge entry -> invoke probability is 0x7fffffff / 0x80000000 = 100.00% [HOT edge]
-;CHECK: edge invoke -> invoke.cont.unreached probability is 0x00000000 / 0x80000000 = 0.00%
-;CHECK: edge invoke -> land.pad probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;CHECK: edge invoke -> invoke.cont.unreached probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
 ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
 
 entry:
@@ -41,8 +41,8 @@ exit:
 define void @test2(i32 %0) personality ptr @"personality_function" {
 ;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00%
 ;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
-;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
-;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00%
+;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
 ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
 
 entry:
@@ -71,8 +71,8 @@ exit:
 define void @test3(i32 %0) personality ptr @"personality_function" {
 ;CHECK: edge entry -> unreached probability is 0x00000000 / 0x80000000 = 0.00%
 ;CHECK: edge entry -> invoke probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
-;CHECK: edge invoke -> invoke.cont.cold probability is 0x7fff8000 / 0x80000000 = 100.00% [HOT edge]
-;CHECK: edge invoke -> land.pad probability is 0x00008000 / 0x80000000 = 0.00%
+;CHECK: edge invoke -> invoke.cont.cold probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;CHECK: edge invoke -> land.pad probability is 0x00000000 / 0x80000000 = 0.00%
 ;CHECK: edge land.pad -> exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
 entry:
   br i1 undef, label %unreached, label %invoke
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll
index c2aa705d3495793..562d979ed0b7e63 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll
@@ -499,8 +499,8 @@ loop:
   %i.0 = phi i32 [ 0, %entry ], [ %inc, %invoke.cont ]
   invoke i32 @InvokeCall()
           to label %invoke.cont unwind label %lpad
-; CHECK: edge loop -> invoke.cont probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
-; CHECK: edge loop -> lpad probability is 0x00000800 / 0x80000000 = 0.00%
+; CHECK: edge loop -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK: edge loop -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
 
 invoke.cont:
   %inc = add nsw i32 %i.0, 1
diff --git a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
index e4bcdfc082ff917..028f1070b8dc5f9 100644
--- a/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
+++ b/llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
@@ -118,8 +118,8 @@ if.then:                                          ; preds = %entry
   %exception = call ptr @__cxa_allocate_exception(i64 1) #0
   invoke i32 @smallFunction(i32 %idx)
           to label %invoke.cont unwind label %lpad
-; CHECK: edge if.then -> invoke.cont probability is 0x40000000 / 0x80000000 = 50.00%
-; CHECK: edge if.then -> lpad probability is 0x40000000 / 0x80000000 = 50.00%
+; CHECK: edge if.then -> invoke.cont probability is 0x80000000 / 0x80000000 = 100.00%
+; CHECK: edge if.then -> lpad probability is 0x00000000 / 0x80000000 = 0.00%
 
 invoke.cont:                                      ; preds = %if.then
   call void @__cxa_throw(ptr %exception, ptr @_ZTIi, ptr null) #1



More information about the llvm-commits mailing list