[llvm] [InstCombine] Update BranchProbabilityAnalysis cache result (PR #86470)
Haohai Wen via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 19 02:29:08 PDT 2024
https://github.com/HaohaiWen updated https://github.com/llvm/llvm-project/pull/86470
>From e80e7433137f37fe6885faac1802be19acfbf3d3 Mon Sep 17 00:00:00 2001
From: Haohai Wen <haohai.wen at intel.com>
Date: Mon, 25 Mar 2024 15:03:30 +0800
Subject: [PATCH 1/3] [InstCombine] Add test to track BPI when condition was
inverted
InstCombine may invert branch condition and profile metadata. Branch
probability analysis should not be preserved in this case.
---
.../Transforms/InstCombine/abandon-bpi.ll | 36 +++++++++++++++++++
1 file changed, 36 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/abandon-bpi.ll
diff --git a/llvm/test/Transforms/InstCombine/abandon-bpi.ll b/llvm/test/Transforms/InstCombine/abandon-bpi.ll
new file mode 100644
index 00000000000000..190a686c8f078b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/abandon-bpi.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -passes="print<branch-prob>,instcombine,print<branch-prob>" -S 2>&1 | FileCheck %s
+
+; CHECK: Printing analysis 'Branch Probability Analysis' for function 'invert_cond':
+; CHECK-NEXT: ---- Branch Probabilities ----
+; CHECK-NEXT: edge %entry -> %bb1 probability is 0x06186186 / 0x80000000 = 4.76%
+; CHECK-NEXT: edge %entry -> %bb2 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge]
+; CHECK-NEXT: Printing analysis 'Branch Probability Analysis' for function 'invert_cond':
+; CHECK-NEXT: ---- Branch Probabilities ----
+; CHECK-NEXT: edge %entry -> %bb2 probability is 0x06186186 / 0x80000000 = 4.76%
+; CHECK-NEXT: edge %entry -> %bb1 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge]
+
+define i32 @invert_cond(ptr %p) {
+; CHECK-LABEL: define i32 @invert_cond(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[COND_NOT:%.*]] = icmp eq ptr [[P]], null
+; CHECK-NEXT: br i1 [[COND_NOT]], label [[BB2:%.*]], label [[BB1:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: bb1:
+; CHECK-NEXT: ret i32 0
+; CHECK: bb2:
+; CHECK-NEXT: ret i32 1
+;
+entry:
+ %cond = icmp ne ptr %p, null
+ br i1 %cond, label %bb1, label %bb2, !prof !1
+
+bb1:
+ ret i32 0;
+
+bb2:
+ ret i32 1;
+}
+
+!1 = !{!"branch_weights", i32 1, i32 20}
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 20, i32 1}
+
>From aa935d02e3a20cf25c6376e4bac2527b27d7ff52 Mon Sep 17 00:00:00 2001
From: Haohai Wen <haohai.wen at intel.com>
Date: Mon, 25 Mar 2024 15:31:15 +0800
Subject: [PATCH 2/3] [InstCombine] Abandon BranchProbabilityAnalysis cache
result
InstCombine may invert branch condition and profile metadata. In such
case, BranchProbabilityAnalysis should be rerun.
---
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 1 +
llvm/test/Transforms/InstCombine/abandon-bpi.ll | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 7c40fb4fc86082..ff7b0b917b712a 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -5314,6 +5314,7 @@ PreservedAnalyses InstCombinePass::run(Function &F,
// Mark all the analyses that instcombine updates as preserved.
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
+ PA.abandon<BranchProbabilityAnalysis>();
return PA;
}
diff --git a/llvm/test/Transforms/InstCombine/abandon-bpi.ll b/llvm/test/Transforms/InstCombine/abandon-bpi.ll
index 190a686c8f078b..fadb2ab16bff6f 100644
--- a/llvm/test/Transforms/InstCombine/abandon-bpi.ll
+++ b/llvm/test/Transforms/InstCombine/abandon-bpi.ll
@@ -6,8 +6,8 @@
; CHECK-NEXT: edge %entry -> %bb2 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge]
; CHECK-NEXT: Printing analysis 'Branch Probability Analysis' for function 'invert_cond':
; CHECK-NEXT: ---- Branch Probabilities ----
-; CHECK-NEXT: edge %entry -> %bb2 probability is 0x06186186 / 0x80000000 = 4.76%
-; CHECK-NEXT: edge %entry -> %bb1 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge]
+; CHECK-NEXT: edge %entry -> %bb2 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge]
+; CHECK-NEXT: edge %entry -> %bb1 probability is 0x06186186 / 0x80000000 = 4.76%
define i32 @invert_cond(ptr %p) {
; CHECK-LABEL: define i32 @invert_cond(
>From 40afcdcd7671c959fe11399f2ca2f463647dd96e Mon Sep 17 00:00:00 2001
From: Haohai Wen <haohai.wen at intel.com>
Date: Thu, 18 Apr 2024 16:31:41 +0800
Subject: [PATCH 3/3] Address comments
---
.../Transforms/InstCombine/InstCombiner.h | 7 +++--
.../InstCombine/InstCombineInternal.h | 6 ++--
.../InstCombine/InstructionCombining.cpp | 30 ++++++++++++++-----
.../{abandon-bpi.ll => update-bpi.ll} | 0
4 files changed, 30 insertions(+), 13 deletions(-)
rename llvm/test/Transforms/InstCombine/{abandon-bpi.ll => update-bpi.ll} (100%)
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index 93090431cbb69f..ea1f4fc3b85dc8 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -76,6 +76,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
SimplifyQuery SQ;
OptimizationRemarkEmitter &ORE;
BlockFrequencyInfo *BFI;
+ BranchProbabilityInfo *BPI;
ProfileSummaryInfo *PSI;
DomConditionCache DC;
@@ -96,13 +97,13 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
DominatorTree &DT, OptimizationRemarkEmitter &ORE,
- BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
- const DataLayout &DL, LoopInfo *LI)
+ BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI,
+ ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI)
: TTI(TTI), Builder(Builder), Worklist(Worklist),
MinimizeSize(MinimizeSize), AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL),
SQ(DL, &TLI, &DT, &AC, nullptr, /*UseInstrInfo*/ true,
/*CanUseUndef*/ true, &DC),
- ORE(ORE), BFI(BFI), PSI(PSI), LI(LI) {}
+ ORE(ORE), BFI(BFI), BPI(BPI), PSI(PSI), LI(LI) {}
virtual ~InstCombiner() = default;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index b9ad3a74007929..a9665269c70c4c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -67,10 +67,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
DominatorTree &DT, OptimizationRemarkEmitter &ORE,
- BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
- const DataLayout &DL, LoopInfo *LI)
+ BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI,
+ ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI)
: InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE,
- BFI, PSI, DL, LI) {}
+ BFI, BPI, PSI, DL, LI) {}
virtual ~InstCombinerImpl() = default;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 9ac146cbfcc909..985c5661467623 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1347,9 +1347,13 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) {
SI->swapProfMetadata();
break;
}
- case Instruction::Br:
- cast<BranchInst>(U)->swapSuccessors(); // swaps prof metadata too
+ case Instruction::Br: {
+ BranchInst *BI = cast<BranchInst>(U);
+ BI->swapSuccessors(); // swaps prof metadata too
+ if (BPI)
+ BPI->swapSuccEdgesProbabilities(BI->getParent());
break;
+ }
case Instruction::Xor:
replaceInstUsesWith(cast<Instruction>(*U), I);
// Add to worklist for DCE.
@@ -3525,6 +3529,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
if (match(Cond, m_Not(m_Value(X))) && !isa<Constant>(X)) {
// Swap Destinations and condition...
BI.swapSuccessors();
+ if (BPI)
+ BPI->swapSuccEdgesProbabilities(BI.getParent());
return replaceOperand(BI, 0, X);
}
@@ -3538,6 +3544,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
Value *NotX = Builder.CreateNot(X, "not." + X->getName());
Value *Or = Builder.CreateLogicalOr(NotX, Y);
BI.swapSuccessors();
+ if (BPI)
+ BPI->swapSuccEdgesProbabilities(BI.getParent());
return replaceOperand(BI, 0, Or);
}
@@ -3554,6 +3562,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
auto *Cmp = cast<CmpInst>(Cond);
Cmp->setPredicate(CmpInst::getInversePredicate(Pred));
BI.swapSuccessors();
+ if (BPI)
+ BPI->swapSuccEdgesProbabilities(BI.getParent());
Worklist.push(Cmp);
return &BI;
}
@@ -5248,7 +5258,8 @@ static bool combineInstructionsOverFunction(
Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA,
AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
- ProfileSummaryInfo *PSI, LoopInfo *LI, const InstCombineOptions &Opts) {
+ BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, LoopInfo *LI,
+ const InstCombineOptions &Opts) {
auto &DL = F.getParent()->getDataLayout();
/// Builder - This is an IRBuilder that automatically inserts new
@@ -5286,7 +5297,7 @@ static bool combineInstructionsOverFunction(
<< F.getName() << "\n");
InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT,
- ORE, BFI, PSI, DL, LI);
+ ORE, BFI, BPI, PSI, DL, LI);
IC.MaxArraySizeForCombine = MaxArraySize;
bool MadeChangeInThisIteration = IC.prepareWorklist(F, RPOT);
MadeChangeInThisIteration |= IC.run();
@@ -5347,16 +5358,16 @@ PreservedAnalyses InstCombinePass::run(Function &F,
MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
auto *BFI = (PSI && PSI->hasProfileSummary()) ?
&AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
+ auto *BPI = AM.getCachedResult<BranchProbabilityAnalysis>(F);
if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
- BFI, PSI, LI, Options))
+ BFI, BPI, PSI, LI, Options))
// No changes, all analyses are preserved.
return PreservedAnalyses::all();
// Mark all the analyses that instcombine updates as preserved.
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
- PA.abandon<BranchProbabilityAnalysis>();
return PA;
}
@@ -5397,9 +5408,14 @@ bool InstructionCombiningPass::runOnFunction(Function &F) {
(PSI && PSI->hasProfileSummary()) ?
&getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() :
nullptr;
+ BranchProbabilityInfo *BPI = nullptr;
+ if (auto *WrapperPass =
+ getAnalysisIfAvailable<BranchProbabilityInfoWrapperPass>())
+ BPI = &WrapperPass->getBPI();
return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
- BFI, PSI, LI, InstCombineOptions());
+ BFI, BPI, PSI, LI,
+ InstCombineOptions());
}
char InstructionCombiningPass::ID = 0;
diff --git a/llvm/test/Transforms/InstCombine/abandon-bpi.ll b/llvm/test/Transforms/InstCombine/update-bpi.ll
similarity index 100%
rename from llvm/test/Transforms/InstCombine/abandon-bpi.ll
rename to llvm/test/Transforms/InstCombine/update-bpi.ll
More information about the llvm-commits
mailing list