[llvm] [BOLT] Include indirect calls/branches in basic block count estimation (PR #225104)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 07:59:56 PDT 2026
https://github.com/andreasmullen created https://github.com/llvm/llvm-project/pull/225104
**Before:** When BOLT adjusts basic block counts in LBR mode it considers outgoing branch counts when `--fix-block-counts` is enabled. This adjustment was made according to the counts of direct calls and branches which are recorded in the `Count` annotation made in `DataReader::convertBranchData`. Indirect calls and branches are also recorded within the `CallProfile` annotation, however they do not contribute to the adjustment of basic block counts.
**After:** When making this adjustment to basic block counts, also consider the counts of indirect calls and branches within the `CallProfile` annotation. Do this by summing the execution counts of the targets from this call/branch and considering them in determining the maximum execution count of the basic block.
Assisted by: Codex. This was to write the tests for `x86` and `RISCV` targets.
>From ae68e2fc24f25d25fdce550a90911b5691135ea7 Mon Sep 17 00:00:00 2001
From: Andreas Mullen <Andreas.Mullen at arm.com>
Date: Fri, 18 Sep 2026 10:15:56 +0000
Subject: [PATCH] [BOLT] Include indirect calls/branches in basic block
estimation
---
bolt/lib/Core/BinaryFunctionProfile.cpp | 27 ++++---
bolt/test/AArch64/fix-block-count-multiple.s | 78 +++++++++++++++++++
.../fix-block-counts-indirect-branch.s | 66 ++++++++++++++++
.../AArch64/fix-block-counts-indirect-call.s | 71 +++++++++++++++++
bolt/test/RISCV/fix-block-count-multiple.s | 78 +++++++++++++++++++
.../RISCV/fix-block-counts-indirect-branch.s | 66 ++++++++++++++++
.../RISCV/fix-block-counts-indirect-call.s | 71 +++++++++++++++++
bolt/test/X86/fix-block-count-multiple.s | 76 ++++++++++++++++++
.../X86/fix-block-counts-indirect-branch.s | 66 ++++++++++++++++
.../test/X86/fix-block-counts-indirect-call.s | 69 ++++++++++++++++
10 files changed, 659 insertions(+), 9 deletions(-)
create mode 100644 bolt/test/AArch64/fix-block-count-multiple.s
create mode 100644 bolt/test/AArch64/fix-block-counts-indirect-branch.s
create mode 100644 bolt/test/AArch64/fix-block-counts-indirect-call.s
create mode 100644 bolt/test/RISCV/fix-block-count-multiple.s
create mode 100644 bolt/test/RISCV/fix-block-counts-indirect-branch.s
create mode 100644 bolt/test/RISCV/fix-block-counts-indirect-call.s
create mode 100644 bolt/test/X86/fix-block-count-multiple.s
create mode 100644 bolt/test/X86/fix-block-counts-indirect-branch.s
create mode 100644 bolt/test/X86/fix-block-counts-indirect-call.s
diff --git a/bolt/lib/Core/BinaryFunctionProfile.cpp b/bolt/lib/Core/BinaryFunctionProfile.cpp
index 94091063833c8..d30d08021f3c3 100644
--- a/bolt/lib/Core/BinaryFunctionProfile.cpp
+++ b/bolt/lib/Core/BinaryFunctionProfile.cpp
@@ -131,17 +131,26 @@ void BinaryFunction::postProcessProfile() {
}
++SuccBIIter;
}
- // Make sure that execution count of a block is at least the number of
- // function calls from the block.
- for (MCInst &Inst : *BB) {
- // Ignore non-call instruction
- if (!BC.MIB->isCall(Inst))
- continue;
- auto CountAnnt = BC.MIB->tryGetAnnotationAs<uint64_t>(Inst, "Count");
- if (CountAnnt)
- BB->setExecutionCount(std::max(BB->getExecutionCount(), *CountAnnt));
+ // Set the execution count of the basic block to be the maximum execution
+ // count across the indirect branches, indirect calls and call
+ // instructions. All other instructions can be ignored.
+ uint64_t MaxCount = BB->getExecutionCount();
+ for (MCInst &Inst : *BB) {
+ uint64_t ExecCount = 0;
+ if (BC.MIB->isIndirectBranch(Inst) || BC.MIB->isIndirectCall(Inst)) {
+ if (auto ICSP = BC.MIB->tryGetAnnotationAs<IndirectCallSiteProfile>(
+ Inst, "CallProfile")) {
+ for (IndirectCallProfile &Entry : *ICSP)
+ ExecCount += Entry.Count;
+ }
+ } else if (BC.MIB->isCall(Inst)) {
+ if (auto Count = BC.MIB->tryGetAnnotationAs<uint64_t>(Inst, "Count"))
+ ExecCount = *Count;
+ }
+ MaxCount = std::max(MaxCount, ExecCount);
}
+ BB->setExecutionCount(MaxCount);
}
}
diff --git a/bolt/test/AArch64/fix-block-count-multiple.s b/bolt/test/AArch64/fix-block-count-multiple.s
new file mode 100644
index 0000000000000..36e6d3c4d86b6
--- /dev/null
+++ b/bolt/test/AArch64/fix-block-count-multiple.s
@@ -0,0 +1,78 @@
+## Check that BOLT correctly adjusts Basic Block weights when given multiple indirect
+## and direct calls with varying execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.icl.fdata FDATA_INDIRECT_CALL_LARGER
+# RUN: link_fdata %s %t.o %t.dcl.fdata FDATA_DIRECT_CALL_LARGER
+# RUN: llvm-strip --strip-symbol=firstIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=secondIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=directLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.icl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.icl.fdata 2>&1 | FileCheck %s --check-prefix=INDIRECT_HIGHER
+# RUN: llvm-bolt %t.exe -o %t.dcl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.dcl.fdata 2>&1 | FileCheck %s --check-prefix=DIRECT_HIGHER
+
+# FDATA_INDIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 40
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 20
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 10
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 30
+# FDATA_INDIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 20
+
+# INDIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# INDIRECT_HIGHER: Entry Point
+# INDIRECT_HIGHER-NEXT: Exec Count : 60{{$}}
+# INDIRECT_HIGHER: blr {{.*}}# CallProfile: 60 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# INDIRECT_HIGHER: blr {{.*}}# CallProfile: 40 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# INDIRECT_HIGHER: bl {{.*}}foo # Count: 20
+
+# FDATA_DIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 10
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 30
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 40
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 20
+# FDATA_DIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 100
+
+# DIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# DIRECT_HIGHER: Entry Point
+# DIRECT_HIGHER-NEXT: Exec Count : 100{{$}}
+# DIRECT_HIGHER: {{.*}}# CallProfile: 40 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# DIRECT_HIGHER: {{.*}}# CallProfile: 60 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# DIRECT_HIGHER: bl {{.*}}foo # Count: 100
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ stp x29, x30, [sp,#-16]!
+ mov x29, sp
+ adr x0, foo
+ adr x1, bar
+firstIndirectLabel:
+ blr x0
+secondIndirectLabel:
+ blr x1
+directLabel:
+ bl foo
+ mov sp, x29
+ ldp x29, x30, [sp], #16
+ ret
+ .size main, .-main
diff --git a/bolt/test/AArch64/fix-block-counts-indirect-branch.s b/bolt/test/AArch64/fix-block-counts-indirect-branch.s
new file mode 100644
index 0000000000000..15df6faeddbda
--- /dev/null
+++ b/bolt/test/AArch64/fix-block-counts-indirect-branch.s
@@ -0,0 +1,66 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect branches
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_BRANCH_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_BRANCH_COUNT
+# RUN: llvm-strip --strip-symbol=branchLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: br {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 30
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: br {{.*}}# TAILCALL {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: br {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ adr x0, foo
+branchLabel:
+ br x0
+ .size main, .-main
diff --git a/bolt/test/AArch64/fix-block-counts-indirect-call.s b/bolt/test/AArch64/fix-block-counts-indirect-call.s
new file mode 100644
index 0000000000000..eca708b4fb91a
--- /dev/null
+++ b/bolt/test/AArch64/fix-block-counts-indirect-call.s
@@ -0,0 +1,71 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect calls
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_CALL_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_CALL_COUNT
+# RUN: llvm-strip --strip-symbol=callLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: blr {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_CALL_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 30
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: blr {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_CALL_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: blr {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ stp x29, x30, [sp,#-16]!
+ mov x29, sp
+ adr x0, foo
+callLabel:
+ blr x0
+ mov sp, x29
+ ldp x29, x30, [sp], #16
+ ret
+ .size main, .-main
diff --git a/bolt/test/RISCV/fix-block-count-multiple.s b/bolt/test/RISCV/fix-block-count-multiple.s
new file mode 100644
index 0000000000000..57c9d6689fd52
--- /dev/null
+++ b/bolt/test/RISCV/fix-block-count-multiple.s
@@ -0,0 +1,78 @@
+## Check that BOLT correctly adjusts Basic Block weights when given multiple indirect
+## and direct calls with varying execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.icl.fdata FDATA_INDIRECT_CALL_LARGER
+# RUN: link_fdata %s %t.o %t.dcl.fdata FDATA_DIRECT_CALL_LARGER
+# RUN: llvm-strip --strip-symbol=firstIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=secondIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=directLabel %t.o
+# RUN: %clang %cflags64 -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.icl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.icl.fdata 2>&1 | FileCheck %s --check-prefix=INDIRECT_HIGHER
+# RUN: llvm-bolt %t.exe -o %t.dcl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.dcl.fdata 2>&1 | FileCheck %s --check-prefix=DIRECT_HIGHER
+
+# FDATA_INDIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 40
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 20
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 10
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 30
+# FDATA_INDIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 20
+
+# INDIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# INDIRECT_HIGHER: Entry Point
+# INDIRECT_HIGHER-NEXT: Exec Count : 60{{$}}
+# INDIRECT_HIGHER: jalr {{.*}}# CallProfile: 60 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# INDIRECT_HIGHER: jalr {{.*}}# CallProfile: 40 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# INDIRECT_HIGHER: jal {{.*}}foo # Count: 20
+
+# FDATA_DIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 10
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 30
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 40
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 20
+# FDATA_DIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 100
+
+# DIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# DIRECT_HIGHER: Entry Point
+# DIRECT_HIGHER-NEXT: Exec Count : 100{{$}}
+# DIRECT_HIGHER: {{.*}}# CallProfile: 40 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# DIRECT_HIGHER: {{.*}}# CallProfile: 60 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# DIRECT_HIGHER: jal {{.*}}foo # Count: 100
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ addi sp, sp, -16
+ sd ra, 8(sp)
+ lla a0, foo
+ lla a1, bar
+firstIndirectLabel:
+ jalr a0
+secondIndirectLabel:
+ jalr a1
+directLabel:
+ jal ra, foo
+ ld ra, 8(sp)
+ addi sp, sp, 16
+ ret
+ .size main, .-main
diff --git a/bolt/test/RISCV/fix-block-counts-indirect-branch.s b/bolt/test/RISCV/fix-block-counts-indirect-branch.s
new file mode 100644
index 0000000000000..a5709329cd3d1
--- /dev/null
+++ b/bolt/test/RISCV/fix-block-counts-indirect-branch.s
@@ -0,0 +1,66 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect branches
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_BRANCH_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_BRANCH_COUNT
+# RUN: llvm-strip --strip-symbol=branchLabel %t.o
+# RUN: %clang %cflags64 -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: jr {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 30
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: jr {{.*}}# TAILCALL {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: jr {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ lla a0, foo
+branchLabel:
+ jr a0
+ .size main, .-main
diff --git a/bolt/test/RISCV/fix-block-counts-indirect-call.s b/bolt/test/RISCV/fix-block-counts-indirect-call.s
new file mode 100644
index 0000000000000..0574a1f04b7d7
--- /dev/null
+++ b/bolt/test/RISCV/fix-block-counts-indirect-call.s
@@ -0,0 +1,71 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect calls
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_CALL_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_CALL_COUNT
+# RUN: llvm-strip --strip-symbol=callLabel %t.o
+# RUN: %clang %cflags64 -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: jalr {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_CALL_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 30
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: jalr {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_CALL_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: jalr {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ addi sp, sp, -16
+ sd ra, 8(sp)
+ lla a0, foo
+callLabel:
+ jalr a0
+ ld ra, 8(sp)
+ addi sp, sp, 16
+ ret
+ .size main, .-main
diff --git a/bolt/test/X86/fix-block-count-multiple.s b/bolt/test/X86/fix-block-count-multiple.s
new file mode 100644
index 0000000000000..b88fa678b554d
--- /dev/null
+++ b/bolt/test/X86/fix-block-count-multiple.s
@@ -0,0 +1,76 @@
+## Check that BOLT correctly adjusts Basic Block weights when given multiple indirect
+## and direct calls with varying execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.icl.fdata FDATA_INDIRECT_CALL_LARGER
+# RUN: link_fdata %s %t.o %t.dcl.fdata FDATA_DIRECT_CALL_LARGER
+# RUN: llvm-strip --strip-symbol=firstIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=secondIndirectLabel %t.o
+# RUN: llvm-strip --strip-symbol=directLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.icl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.icl.fdata 2>&1 | FileCheck %s --check-prefix=INDIRECT_HIGHER
+# RUN: llvm-bolt %t.exe -o %t.dcl.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.dcl.fdata 2>&1 | FileCheck %s --check-prefix=DIRECT_HIGHER
+
+# FDATA_INDIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 40
+# FDATA_INDIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 20
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 10
+# FDATA_INDIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 30
+# FDATA_INDIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 20
+
+# INDIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# INDIRECT_HIGHER: Entry Point
+# INDIRECT_HIGHER-NEXT: Exec Count : 60{{$}}
+# INDIRECT_HIGHER: callq {{.*}}# CallProfile: 60 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# INDIRECT_HIGHER: callq {{.*}}# CallProfile: 40 (0 misses) :
+# INDIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# INDIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# INDIRECT_HIGHER: callq {{.*}}foo # Count: 20
+
+# FDATA_DIRECT_CALL_LARGER: 0 [unknown] 0 1 main 0 0 0
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 foo 0 0 10
+# FDATA_DIRECT_CALL_LARGER: 1 main #firstIndirectLabel# 1 bar 0 0 30
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 foo 0 0 40
+# FDATA_DIRECT_CALL_LARGER: 1 main #secondIndirectLabel# 1 bar 0 0 20
+# FDATA_DIRECT_CALL_LARGER: 1 main #directLabel# 1 foo 0 0 100
+
+# DIRECT_HIGHER-LABEL: Binary Function "main" after building cfg
+# DIRECT_HIGHER: Entry Point
+# DIRECT_HIGHER-NEXT: Exec Count : 100{{$}}
+# DIRECT_HIGHER: {{.*}}# CallProfile: 40 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 30 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 10 (0 misses) }
+# DIRECT_HIGHER: {{.*}}# CallProfile: 60 (0 misses) :
+# DIRECT_HIGHER-DAG: { bar: 20 (0 misses) }
+# DIRECT_HIGHER-DAG: { foo: 40 (0 misses) }
+# DIRECT_HIGHER: callq {{.*}}foo # Count: 100
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ pushq %rbp
+ leaq foo(%rip), %rax
+ leaq bar(%rip), %rcx
+firstIndirectLabel:
+ callq *%rax
+secondIndirectLabel:
+ callq *%rcx
+directLabel:
+ callq foo
+ popq %rbp
+ ret
+ .size main, .-main
diff --git a/bolt/test/X86/fix-block-counts-indirect-branch.s b/bolt/test/X86/fix-block-counts-indirect-branch.s
new file mode 100644
index 0000000000000..dfd9574810b4e
--- /dev/null
+++ b/bolt/test/X86/fix-block-counts-indirect-branch.s
@@ -0,0 +1,66 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect branches
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_BRANCH_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_BRANCH_COUNT
+# RUN: llvm-strip --strip-symbol=branchLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: jmpl {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 30
+# FDATA_BELOW_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: jmpl {{.*}}# TAILCALL {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_BRANCH_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 foo 0 0 20
+# FDATA_ABOVE_BRANCH_COUNT: 1 main #branchLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: jmpl {{.*}}# TAILCALL {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ leaq foo(%rip), %rax
+branchLabel:
+ jmpq *%rax
+ .size main, .-main
diff --git a/bolt/test/X86/fix-block-counts-indirect-call.s b/bolt/test/X86/fix-block-counts-indirect-call.s
new file mode 100644
index 0000000000000..05beebe5b97e4
--- /dev/null
+++ b/bolt/test/X86/fix-block-counts-indirect-call.s
@@ -0,0 +1,69 @@
+## Check that BOLT correctly adjusts Basic Block weights when given indirect calls
+## and a variety of different execution counts in the profile.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.zcc.fdata FDATA_ZERO_BB_COUNT
+# RUN: link_fdata %s %t.o %t.bcc.fdata FDATA_BELOW_CALL_COUNT
+# RUN: link_fdata %s %t.o %t.acc.fdata FDATA_ABOVE_CALL_COUNT
+# RUN: llvm-strip --strip-symbol=callLabel %t.o
+# RUN: %clang %cflags -Wl,-q %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.zcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.zcc.fdata 2>&1 | FileCheck %s --check-prefix=ZERO
+# RUN: llvm-bolt %t.exe -o %t.bcc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.bcc.fdata 2>&1 | FileCheck %s --check-prefix=BELOW
+# RUN: llvm-bolt %t.exe -o %t.acc.bolt --fix-block-counts --print-cfg \
+# RUN: --data=%t.acc.fdata 2>&1 | FileCheck %s --check-prefix=ABOVE
+
+# FDATA_ZERO_BB_COUNT: 0 [unknown] 0 1 main 0 0 0
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ZERO_BB_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ZERO-LABEL: Binary Function "main" after building cfg
+# ZERO: Entry Point
+# ZERO-NEXT: Exec Count : 50{{$}}
+# ZERO: callq {{.*}}# CallProfile: 50 (0 misses) :
+# ZERO-DAG: { foo: 20 (0 misses) }
+# ZERO-DAG: { bar: 30 (0 misses) }
+
+# FDATA_BELOW_CALL_COUNT: 0 [unknown] 0 1 main 0 0 10
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 30
+# FDATA_BELOW_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 70
+
+# BELOW-LABEL: Binary Function "main" after building cfg
+# BELOW: Entry Point
+# BELOW-NEXT: Exec Count : 100{{$}}
+# BELOW: callq {{.*}}# CallProfile: 100 (0 misses) :
+# BELOW-DAG: { foo: 30 (0 misses) }
+# BELOW-DAG: { bar: 70 (0 misses) }
+
+# FDATA_ABOVE_CALL_COUNT: 0 [unknown] 0 1 main 0 0 90
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 foo 0 0 20
+# FDATA_ABOVE_CALL_COUNT: 1 main #callLabel# 1 bar 0 0 30
+
+# ABOVE-LABEL: Function "main" after building cfg
+# ABOVE: Entry Point
+# ABOVE-NEXT: Exec Count : 90{{$}}
+# ABOVE: callq {{.*}}# CallProfile: 50 (0 misses) :
+# ABOVE-DAG: { foo: 20 (0 misses) }
+# ABOVE-DAG: { bar: 30 (0 misses) }
+
+ .type main, at function
+ .type foo, at function
+ .type bar, at function
+ .globl main,foo,bar
+foo:
+ ret
+ .size foo, .-foo
+
+bar:
+ ret
+ .size bar, .-bar
+
+main:
+ pushq %rbp
+ leaq foo(%rip), %rax
+callLabel:
+ callq *%rax
+ popq %rbp
+ ret
+ .size main, .-main
More information about the llvm-commits
mailing list