[llvm] [SimplifyCFG] Prevent merging cbranch to cbranch if the branch probability from the first to second is too low. (PR #69375)
Valery Pykhtin via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 17 12:37:55 PDT 2023
https://github.com/vpykhtin created https://github.com/llvm/llvm-project/pull/69375
AMDGPU target has faced the situation which can be illustrated with the following testcase:
```
define void @dont_merge_cbranches(i32 %V) {
%divergent_cond = icmp ne i32 %V, 0
%uniform_cond = call i1 @uniform_result(i1 %divergent_cond)
br i1 %uniform_cond, label %bb2, label %exit, !prof !0
bb2:
br i1 %divergent_cond, label %bb3, label %exit
bb3:
call void @bar( )
br label %exit
exit:
ret void
}
!0 = !{!"branch_weights", i32 1, i32 100000}
```
SimplifyCFG merges branches on %uniform_cond and %divergent_cond which is undesirable because the first branch to _bb2_ is taken extremely rare and the second branch is expensive (as names state the first branch is uniform and the second is divergent and requires EXEC mask handling).
@nhaehnle suggested to stop merging such branches on the basis of branch probability information.
Threshold value is somewhat voluntaristic at the moment and probably should be selected as the lowest from the known used branch probability values in the LLVM code to preserve current behaviour. It may also happen that we end up using target dependent threshold.
>From fc1457d4856b3f4ae0a080a451669e344b0b1354 Mon Sep 17 00:00:00 2001
From: Valery Pykhtin <valery.pykhtin at gmail.com>
Date: Tue, 17 Oct 2023 20:50:48 +0200
Subject: [PATCH 1/2] [SimplifyCFG] Add test on prevent merging cbranch to
cbranch if the branch probabililty from the first to second is too low.
---
.../SimplifyCFG/branch-cond-dont-merge.ll | 59 +++++++++++++++++++
1 file changed, 59 insertions(+)
create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll b/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
new file mode 100644
index 000000000000000..b62bb825ab30b62
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-cbranch-to-cbranch-weight-ratio=100 -S | FileCheck %s
+
+declare void @bar()
+declare i1 @uniform_result(i1 %c)
+
+define void @dont_merge_cbranches1(i32 %V) {
+; CHECK-LABEL: @dont_merge_cbranches1(
+; CHECK-NEXT: [[DIVERGENT_COND:%.*]] = icmp ne i32 [[V:%.*]], 0
+; CHECK-NEXT: [[UNIFORM_COND:%.*]] = call i1 @uniform_result(i1 [[DIVERGENT_COND]])
+; CHECK-NEXT: [[UNIFORM_COND_NOT:%.*]] = xor i1 [[UNIFORM_COND]], true
+; CHECK-NEXT: [[DIVERGENT_COND_NOT:%.*]] = xor i1 [[DIVERGENT_COND]], true
+; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[UNIFORM_COND_NOT]], i1 true, i1 [[DIVERGENT_COND_NOT]]
+; CHECK-NEXT: br i1 [[BRMERGE]], label [[EXIT:%.*]], label [[BB3:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: bb3:
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+ %divergent_cond = icmp ne i32 %V, 0
+ %uniform_cond = call i1 @uniform_result(i1 %divergent_cond)
+ br i1 %uniform_cond, label %bb2, label %exit, !prof !0
+bb2:
+ br i1 %divergent_cond, label %bb3, label %exit
+bb3:
+ call void @bar( )
+ br label %exit
+exit:
+ ret void
+}
+
+define void @dont_merge_cbranches2(i32 %V) {
+; CHECK-LABEL: @dont_merge_cbranches2(
+; CHECK-NEXT: [[DIVERGENT_COND:%.*]] = icmp ne i32 [[V:%.*]], 0
+; CHECK-NEXT: [[UNIFORM_COND:%.*]] = call i1 @uniform_result(i1 [[DIVERGENT_COND]])
+; CHECK-NEXT: [[DIVERGENT_COND_NOT:%.*]] = xor i1 [[DIVERGENT_COND]], true
+; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[UNIFORM_COND]], i1 true, i1 [[DIVERGENT_COND_NOT]]
+; CHECK-NEXT: br i1 [[BRMERGE]], label [[EXIT:%.*]], label [[BB3:%.*]], !prof [[PROF0]]
+; CHECK: bb3:
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+ %divergent_cond = icmp ne i32 %V, 0
+ %uniform_cond = call i1 @uniform_result(i1 %divergent_cond)
+ br i1 %uniform_cond, label %exit, label %bb2, !prof !1
+bb2:
+ br i1 %divergent_cond, label %bb3, label %exit
+bb3:
+ call void @bar( )
+ br label %exit
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1000}
+!1 = !{!"branch_weights", i32 1000, i32 1}
>From d5a0880178beb2be8d22e049a607f94173a00903 Mon Sep 17 00:00:00 2001
From: Valery Pykhtin <valery.pykhtin at gmail.com>
Date: Tue, 17 Oct 2023 20:53:04 +0200
Subject: [PATCH 2/2] [SimplifyCFG] Prevent merging cbranch to cbranch if the
branch probabililty from the first to second is too low.
---
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 14 ++++++++++++++
.../SimplifyCFG/branch-cond-dont-merge.ll | 13 ++++++-------
2 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 35fead111aa9666..06e0fc91483a514 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -180,6 +180,11 @@ static cl::opt<unsigned> MaxSwitchCasesPerResult(
"max-switch-cases-per-result", cl::Hidden, cl::init(16),
cl::desc("Limit cases to analyze when converting a switch to select"));
+static cl::opt<unsigned> CondBranchToCondBranchWeightRatio(
+ "simplifycfg-cbranch-to-cbranch-weight-ratio", cl::Hidden, cl::init(10000),
+ cl::desc("Don't merge conditional branches if the branch probability from "
+ "the first to second is below of the reciprocal of this value"));
+
STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
STATISTIC(NumLinearMaps,
"Number of switch instructions turned into linear mapping");
@@ -4347,6 +4352,15 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
if (PBI->getSuccessor(PBIOp) == BB)
return false;
+ // If predecessor's branch probability to BB is too low don't merge branches.
+ SmallVector<uint32_t, 2> PredWeights;
+ if (extractBranchWeights(*PBI, PredWeights)) {
+ auto BIWeight = PredWeights[PBIOp ^ 1];
+ auto CommonWeight = PredWeights[PBIOp];
+ if (CommonWeight / BIWeight > CondBranchToCondBranchWeightRatio)
+ return false;
+ }
+
// Do not perform this transformation if it would require
// insertion of a large number of select instructions. For targets
// without predication/cmovs, this is a big pessimization.
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll b/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
index b62bb825ab30b62..6dcdfee21932f12 100644
--- a/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
+++ b/llvm/test/Transforms/SimplifyCFG/branch-cond-dont-merge.ll
@@ -8,10 +8,9 @@ define void @dont_merge_cbranches1(i32 %V) {
; CHECK-LABEL: @dont_merge_cbranches1(
; CHECK-NEXT: [[DIVERGENT_COND:%.*]] = icmp ne i32 [[V:%.*]], 0
; CHECK-NEXT: [[UNIFORM_COND:%.*]] = call i1 @uniform_result(i1 [[DIVERGENT_COND]])
-; CHECK-NEXT: [[UNIFORM_COND_NOT:%.*]] = xor i1 [[UNIFORM_COND]], true
-; CHECK-NEXT: [[DIVERGENT_COND_NOT:%.*]] = xor i1 [[DIVERGENT_COND]], true
-; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[UNIFORM_COND_NOT]], i1 true, i1 [[DIVERGENT_COND_NOT]]
-; CHECK-NEXT: br i1 [[BRMERGE]], label [[EXIT:%.*]], label [[BB3:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT: br i1 [[UNIFORM_COND]], label [[BB2:%.*]], label [[EXIT:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: bb2:
+; CHECK-NEXT: br i1 [[DIVERGENT_COND]], label [[BB3:%.*]], label [[EXIT]]
; CHECK: bb3:
; CHECK-NEXT: call void @bar()
; CHECK-NEXT: br label [[EXIT]]
@@ -34,9 +33,9 @@ define void @dont_merge_cbranches2(i32 %V) {
; CHECK-LABEL: @dont_merge_cbranches2(
; CHECK-NEXT: [[DIVERGENT_COND:%.*]] = icmp ne i32 [[V:%.*]], 0
; CHECK-NEXT: [[UNIFORM_COND:%.*]] = call i1 @uniform_result(i1 [[DIVERGENT_COND]])
-; CHECK-NEXT: [[DIVERGENT_COND_NOT:%.*]] = xor i1 [[DIVERGENT_COND]], true
-; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[UNIFORM_COND]], i1 true, i1 [[DIVERGENT_COND_NOT]]
-; CHECK-NEXT: br i1 [[BRMERGE]], label [[EXIT:%.*]], label [[BB3:%.*]], !prof [[PROF0]]
+; CHECK-NEXT: br i1 [[UNIFORM_COND]], label [[EXIT:%.*]], label [[BB2:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: bb2:
+; CHECK-NEXT: br i1 [[DIVERGENT_COND]], label [[BB3:%.*]], label [[EXIT]]
; CHECK: bb3:
; CHECK-NEXT: call void @bar()
; CHECK-NEXT: br label [[EXIT]]
More information about the llvm-commits
mailing list