[llvm] [BOLT][AArch64] Add call relaxation pass (PR #173952)
Rafael Auler via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 16:56:18 PDT 2026
https://github.com/rafaelauler updated https://github.com/llvm/llvm-project/pull/173952
>From 60fd85beb18552e7305845a34a975c720eef6175 Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks at meta.com>
Date: Mon, 15 Dec 2025 23:06:58 -0800
Subject: [PATCH 1/3] [BOLT][AArch64] Add experimental call relaxation pass
Add a function call relaxation pass that groups functions into clusters,
each with a maximum size of 128MB. This pass is enabled using the
--relax-exp` option.
* Within each cluster: function calls do not require relaxation.
* Between clusters: thunks are created at the cluster boundaries to
handle inter-cluster calls. These thunks can be either short or long,
depending on the proximity of the thunk destination. When possible,
a long thunk may be shared by both adjacent clusters.
* Hot functions: if all hot functions (after reordering) fit within a
single cluster (under 128MB), no thunks are needed on the hot path, as
determined by the profile.
* PLT behavior: for the Procedure Linkage Table (PLT), it is currently
assumed that the hottest cluster will be placed adjacent to the PLT, so
thunks are not required for that cluster. This assumption can be
disabled with `--relax-plt=0` option.
* Pass behavior: when enabled, this pass replaces the LongJmp pass. It
typically runs much faster because it does not attempt to predict the
exact code layout.
---
bolt/include/bolt/Core/BinaryContext.h | 12 +-
bolt/include/bolt/Passes/LongJmp.h | 41 ++++
bolt/lib/Core/BinaryContext.cpp | 10 +-
bolt/lib/Passes/BinaryPasses.cpp | 8 +-
bolt/lib/Passes/LongJmp.cpp | 313 ++++++++++++++++++++++++-
bolt/lib/Rewrite/BinaryPassManager.cpp | 6 +-
bolt/test/AArch64/relax-exp.s | 78 ++++++
7 files changed, 457 insertions(+), 11 deletions(-)
create mode 100644 bolt/test/AArch64/relax-exp.s
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 31c90d2c502bd..968a648238c8a 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -558,7 +558,14 @@ class BinaryContext {
}
/// Return functions meant for the output in a sorted order.
- BinaryFunctionListType &getOutputBinaryFunctions() { return OutputFunctions; }
+ const BinaryFunctionListType &getOutputBinaryFunctions() const {
+ return OutputFunctions;
+ }
+
+ /// Update output function list.
+ void updateOutputBinaryFunctions(BinaryFunctionListType &&Functions) {
+ OutputFunctions.swap(Functions);
+ }
/// Create BOLT-injected function
BinaryFunction *createInjectedBinaryFunction(const std::string &Name,
@@ -576,6 +583,9 @@ class BinaryContext {
const InstructionListType &Instructions,
const Twine &Name = "");
+ /// Create a binary function with a base \p Name.
+ BinaryFunction *createThunkBinaryFunction(const std::string &Name);
+
BinaryFunctionListType &getInjectedBinaryFunctions() {
return InjectedBinaryFunctions;
}
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 4b4935888599a..702f942ab260e 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -76,6 +76,47 @@ class LongJmpPass : public BinaryFunctionPass {
/// 128MB of each other.
void relaxLocalBranches(BinaryFunction &BF);
+ /// A group of functions that are located within the longest direct
+ /// branch/call instruction distance. Functions withing the cluster do not
+ /// require a thunk for calls with the same cluster. The cluster may include
+ /// a set of thunks for covering calls to functions outside.
+ struct FunctionCluster {
+ /// All functions in this cluster.
+ DenseSet<BinaryFunction *> Functions;
+
+ /// Symbols corresponding to entry points of functions that this cluster
+ /// calls. Note that it excludes all functions in the cluster itself.
+ DenseSet<const MCSymbol *> Callees;
+
+ /// Estimated size of the cluster in bytes.
+ uint64_t Size{0};
+
+ /// The index of the last function in the cluster. Used as an insertion
+ /// point for adding thunks to the output function list.
+ size_t LastFunctionIndex = -1;
+
+ /// When placing hot code at the end of the binary, track the first function
+ /// for insertion purposes.
+ size_t FirstFunctionIndex = -1;
+
+ /// Thunks located at the end of this cluster.
+ BinaryFunctionListType ThunkList;
+
+ /// Thunks used by this cluster. Some could be in a ThunkList of the
+ /// preceding cluster.
+ ///
+ /// <Function Symbol> -> <Thunk Function>.
+ DenseMap<const MCSymbol *, BinaryFunction *> Thunks;
+ };
+
+ /// Maximum size of combined regular functions in the cluster. Note that it's
+ /// less than 128MB, because the size of the cluster plus its thunks should be
+ /// less than 128MB.
+ static constexpr uint64_t MaxClusterSize = 125 * 1024 * 1024;
+
+ /// Relax calls using function cluster approach.
+ void relaxCalls(BinaryContext &BC);
+
/// -- Layout estimation methods --
/// Try to do layout before running the emitter, by looking at BinaryFunctions
/// and MCInsts -- this is an estimation. To be correct for longjmp inserter
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index f0541921c70a8..b51c3b2be4259 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2560,8 +2560,8 @@ BinaryContext::createInjectedBinaryFunction(const std::string &Name,
setSymbolToFunctionMap(BF->getSymbol(), BF);
BF->CurrentState = BinaryFunction::State::CFG;
- if (!getOutputBinaryFunctions().empty())
- getOutputBinaryFunctions().push_back(BF);
+ if (!OutputFunctions.empty())
+ OutputFunctions.push_back(BF);
return BF;
}
@@ -2603,6 +2603,12 @@ BinaryContext::createInstructionPatch(uint64_t Address,
return PBF;
}
+BinaryFunction *
+BinaryContext::createThunkBinaryFunction(const std::string &Name) {
+ static NameResolver NR;
+ return createInjectedBinaryFunction(NR.uniquify(Name));
+}
+
std::pair<size_t, size_t>
BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
// Use the original size for non-simple functions.
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index 480d0cef58f43..984fe9808b95d 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -555,10 +555,10 @@ Error FixupBranches::runOnFunctions(BinaryContext &BC) {
}
Error PopulateOutputFunctions::runOnFunctions(BinaryContext &BC) {
- BinaryFunctionListType &OutputFunctions = BC.getOutputBinaryFunctions();
-
- assert(OutputFunctions.empty() && "Output function list already initialized");
+ assert(BC.getOutputBinaryFunctions().empty() &&
+ "Output function list already initialized");
+ BinaryFunctionListType OutputFunctions;
OutputFunctions.reserve(BC.getBinaryFunctions().size() +
BC.getInjectedBinaryFunctions().size());
llvm::transform(llvm::make_second_range(BC.getBinaryFunctions()),
@@ -586,6 +586,8 @@ Error PopulateOutputFunctions::runOnFunctions(BinaryContext &BC) {
[](const BinaryFunction *A) { return !A->hasValidIndex(); });
}
+ BC.updateOutputBinaryFunctions(std::move(OutputFunctions));
+
return Error::success();
}
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 9fedf919dd489..59d94f2618e65 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -30,6 +30,15 @@ extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool> GroupStubs("group-stubs",
cl::desc("share stubs across functions"),
cl::init(true), cl::cat(BoltOptCategory));
+
+static cl::opt<bool>
+ ExperimentalRelaxation("relax-exp",
+ cl::desc("run experimental relaxation pass"),
+ cl::init(false), cl::cat(BoltOptCategory));
+
+static cl::opt<bool> RelaxPLT("relax-plt",
+ cl::desc("indicate PLT proximity to hot text"),
+ cl::init(true), cl::cat(BoltOptCategory));
}
namespace llvm {
@@ -944,13 +953,307 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) {
}
}
+void LongJmpPass::relaxCalls(BinaryContext &BC) {
+ // Operate on a copy of binary functions. We are going to manually insert new
+ // thunks and update the list.
+ BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
+
+ // Conservatively estimate emitted function size. Assume the worst case
+ // alignment.
+ auto estimateFunctionSize = [&](const BinaryFunction &BF) -> uint64_t {
+ // Conservative estimation of the aligned function size.
+ if (!BC.shouldEmit(BF))
+ return 0;
+ uint64_t Size = BF.estimateSize();
+ if (BF.hasValidIndex())
+ Size += BF.getAlignment();
+
+ if (BF.hasIslandsInfo()) {
+ Size += BF.getConstantIslandAlignment();
+ Size += BF.estimateConstantIslandSize();
+ }
+
+ return Size;
+ };
+
+ // Map every function to its direct callees. Note that this is different from
+ // the regular call graph as here we completely ignore indirect calls.
+ uint64_t EstimatedSize = 0;
+ DenseMap<BinaryFunction *, std::set<const MCSymbol *>> CallMap;
+ for (BinaryFunction *BF : OutputFunctions) {
+ if (!BC.shouldEmit(*BF) || BF->isPatch())
+ continue;
+
+ EstimatedSize += estimateFunctionSize(*BF);
+
+ for (const BinaryBasicBlock &BB : *BF) {
+ for (const MCInst &Inst : BB) {
+ if (!BC.MIB->isCall(Inst) || BC.MIB->isIndirectCall(Inst) ||
+ BC.MIB->isIndirectBranch(Inst))
+ continue;
+ const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
+ assert(TargetSymbol);
+
+ // Ignore internal calls that use basic block labels as a destination.
+ if (!BC.getFunctionForSymbol(TargetSymbol))
+ continue;
+
+ CallMap[BF].insert(TargetSymbol);
+ }
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << EstimatedSize
+ << '\n');
+
+ // Build clusters in the order the functions will appear in the output.
+ std::vector<FunctionCluster> Clusters;
+ for (size_t Index = 0, NumFuncs = OutputFunctions.size(); Index < NumFuncs;
+ ++Index) {
+ const size_t BFIndex =
+ opts::HotFunctionsAtEnd ? NumFuncs - Index - 1 : Index;
+ BinaryFunction *BF = OutputFunctions[BFIndex];
+ if (!BC.shouldEmit(*BF) || BF->isPatch())
+ continue;
+
+ const uint64_t BFSize = estimateFunctionSize(*BF);
+ if (Clusters.empty() || Clusters.back().Size + BFSize > MaxClusterSize) {
+ Clusters.emplace_back(FunctionCluster());
+ Clusters.back().FirstFunctionIndex = BFIndex;
+ }
+
+ FunctionCluster &FC = Clusters.back();
+ FC.Functions.insert(BF);
+
+ // When a function is added to the cluster, we have to remove all of its
+ // symbols from the cluster callee list. These include alternative symbols
+ // (e.g. after ICF) and secondary entry point symbols.
+ for (const MCSymbol *Symbol : BF->getSymbols()) {
+ auto It = FC.Callees.find(Symbol);
+ if (It != FC.Callees.end())
+ FC.Callees.erase(It);
+ }
+ BF->forEachEntryPoint(
+ [&FC](uint64_t Offset, const MCSymbol *EntrySymbol) -> bool {
+ auto It = FC.Callees.find(EntrySymbol);
+ if (It != FC.Callees.end())
+ FC.Callees.erase(It);
+ return true;
+ });
+
+ // Update cluster callee list with added function callees.
+ for (const MCSymbol *CalleeSymbol : CallMap[BF]) {
+ BinaryFunction *Callee = BC.getFunctionForSymbol(CalleeSymbol);
+ if (!FC.Functions.count(Callee)) {
+ FC.Callees.insert(CalleeSymbol);
+ }
+ }
+
+ FC.Size += BFSize;
+ FC.LastFunctionIndex = BFIndex;
+ }
+
+ if (opts::HotFunctionsAtEnd) {
+ std::reverse(Clusters.begin(), Clusters.end());
+ llvm::for_each(Clusters, [](FunctionCluster &FC) {
+ std::swap(FC.LastFunctionIndex, FC.FirstFunctionIndex);
+ });
+ }
+
+ if (Clusters.empty())
+ return;
+
+ // Print cluster stats.
+ BC.outs() << "BOLT-INFO: built " << Clusters.size()
+ << " function cluster(s)\n";
+ uint64_t ClusterIndex = 0;
+ for (const FunctionCluster &FC : Clusters) {
+ BC.outs() << "BOLT-INFO: cluster: " << ClusterIndex++ << '\n'
+ << "BOLT-INFO: " << FC.Functions.size() << " function(s)\n"
+ << "BOLT-INFO: " << FC.Callees.size() << " callee(s)\n"
+ << "BOLT-INFO: " << FC.Size << " estimated bytes\n";
+ }
+
+ if (opts::RelaxPLT) {
+ // Populate one of the clusters with PLT functions based on the proximity of
+ // the PLT section to avoid unneeded thunk redirection.
+ const size_t PLTClusterNum = opts::UseOldText ? Clusters.size() - 1 : 0;
+ auto &PLTCluster = Clusters[PLTClusterNum];
+ for (BinaryFunction &BF :
+ llvm::make_second_range(BC.getBinaryFunctions())) {
+ if (BF.isPLTFunction()) {
+ PLTCluster.Functions.insert(&BF);
+ auto It = PLTCluster.Callees.find(BF.getSymbol());
+ if (It != PLTCluster.Callees.end())
+ PLTCluster.Callees.erase(It);
+ }
+ }
+ }
+
+ /// Create a thunk with +-128MB span.
+ size_t NumShortThunks = 0;
+ auto createShortThunk = [&](const MCSymbol *TargetSymbol) {
+ ++NumShortThunks;
+ BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(
+ "__AArch64Thunk_" + TargetSymbol->getName().str());
+ MCInst Inst;
+ BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
+ ThunkBF->addBasicBlock()->addInstruction(Inst);
+
+ return ThunkBF;
+ };
+
+ /// Create a thunk with +-4GB span.
+ size_t NumLongThunks = 0;
+ auto createLongThunk = [&](const MCSymbol *TargetSymbol) {
+ ++NumLongThunks;
+ BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(
+ "__AArch64ADRPThunk_" + TargetSymbol->getName().str());
+ InstructionListType Instructions;
+ BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
+ ThunkBF->addBasicBlock()->addInstructions(Instructions);
+
+ return ThunkBF;
+ };
+
+ for (unsigned ClusterNum = 0; ClusterNum < Clusters.size(); ++ClusterNum) {
+ FunctionCluster &FC = Clusters[ClusterNum];
+ SmallVector<const MCSymbol *, 16> Callees(FC.Callees.begin(),
+ FC.Callees.end());
+
+ // Generate thunks in deterministic order.
+ llvm::sort(Callees, [&BC](const MCSymbol *A, const MCSymbol *B) {
+ uint64_t EntryA;
+ uint64_t EntryB;
+ BinaryFunction *BFA = BC.getFunctionForSymbol(A, &EntryA);
+ BinaryFunction *BFB = BC.getFunctionForSymbol(B, &EntryB);
+ if (BFA == BFB) {
+ if (EntryA != EntryB)
+ return EntryA < EntryB;
+
+ // Use lexicographical order for ICF'ed symbols.
+ return A->getName() < B->getName();
+ }
+ return compareBinaryFunctionByIndex(BFA, BFB);
+ });
+
+ // Return index of adjacent cluster containing the function.
+ auto getAdjClusterWithFunction =
+ [&](const BinaryFunction *BF) -> std::optional<unsigned> {
+ if (ClusterNum > 0 && Clusters[ClusterNum - 1].Functions.count(BF))
+ return ClusterNum - 1;
+ if (ClusterNum + 1 < Clusters.size() &&
+ Clusters[ClusterNum + 1].Functions.count(BF))
+ return ClusterNum + 1;
+ return std::nullopt;
+ };
+
+ const FunctionCluster *PrevCluster =
+ ClusterNum ? &Clusters[ClusterNum - 1] : nullptr;
+
+ // Create short thunks for callees in adjacent clusters and long thunks
+ // for callees outside.
+ for (const MCSymbol *Callee : Callees) {
+ if (FC.Thunks.count(Callee))
+ continue;
+
+ BinaryFunction *Thunk = 0;
+ std::optional<unsigned> AdjCluster =
+ getAdjClusterWithFunction(BC.getFunctionForSymbol(Callee));
+ if (AdjCluster) {
+ Thunk = createShortThunk(Callee);
+ } else {
+ // Previous cluster may already have a long thunk that can be reused.
+ if (PrevCluster) {
+ auto It = PrevCluster->Thunks.find(Callee);
+ // Reuse only if previous cluster hosts this thunk.
+ if (It != PrevCluster->Thunks.end() &&
+ llvm::is_contained(PrevCluster->ThunkList, It->second)) {
+ FC.Thunks[Callee] = It->second;
+ continue;
+ }
+ }
+ Thunk = createLongThunk(Callee);
+ }
+
+ // The cluster that will host this thunk. If the current cluster is the
+ // last one, try to use the previous one. Matters when we want to have hot
+ // functions at higher addresses under HotFunctionsAtEnd.
+ FunctionCluster *ThunkCluster = &Clusters[ClusterNum];
+ if ((AdjCluster && *AdjCluster == ClusterNum - 1) ||
+ (ClusterNum && ClusterNum == Clusters.size() - 1))
+ ThunkCluster = &Clusters[ClusterNum - 1];
+ ThunkCluster->ThunkList.push_back(Thunk);
+
+ // Register thunks for all symbols associated with the function.
+ uint64_t EntryID = 0;
+ const BinaryFunction *BF = BC.getFunctionForSymbol(Callee, &EntryID);
+ if (EntryID != 0) {
+ FC.Thunks[Callee] = Thunk;
+ } else {
+ for (const MCSymbol *Symbol : BF->getSymbols()) {
+ FC.Thunks[Symbol] = Thunk;
+ }
+ }
+ }
+ }
+
+ if (NumShortThunks)
+ BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
+
+ if (NumLongThunks)
+ BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
+
+ // Replace callees with thunks.
+ for (FunctionCluster &FC : Clusters) {
+ for (BinaryFunction *BF : FC.Functions) {
+ if (!CallMap.count(BF))
+ continue;
+
+ for (BinaryBasicBlock &BB : *BF) {
+ for (MCInst &Inst : BB) {
+ if (!BC.MIB->isCall(Inst) || BC.MIB->isIndirectCall(Inst) ||
+ BC.MIB->isIndirectBranch(Inst))
+ continue;
+ const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
+ assert(TargetSymbol);
+
+ auto It = FC.Thunks.find(TargetSymbol);
+ if (It != FC.Thunks.end())
+ BC.MIB->replaceBranchTarget(Inst, It->second->getSymbol(),
+ BC.Ctx.get());
+ }
+ }
+ }
+ }
+
+ // Add thunks to the function list and assign a section name matching the
+ // function they follow.
+ for (const FunctionCluster &FC : llvm::reverse(Clusters)) {
+ std::string SectionName =
+ OutputFunctions[FC.LastFunctionIndex]->getCodeSectionName().str().str();
+ for (BinaryFunction *Thunk : FC.ThunkList) {
+ Thunk->setCodeSectionName(SectionName);
+ }
+
+ OutputFunctions.insert(
+ std::next(OutputFunctions.begin(), FC.LastFunctionIndex + 1),
+ FC.ThunkList.begin(), FC.ThunkList.end());
+ }
+
+ LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
+ for (const auto *BF : OutputFunctions) { dbgs() << *BF << '\n'; });
+
+ BC.updateOutputBinaryFunctions(std::move(OutputFunctions));
+}
+
Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
- assert((opts::CompactCodeModel ||
+ assert((opts::CompactCodeModel || opts::ExperimentalRelaxation ||
opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) &&
"LongJmp cannot work with functions split in more than two fragments");
- if (opts::CompactCodeModel) {
+ if (opts::CompactCodeModel || opts::ExperimentalRelaxation) {
BC.outs()
<< "BOLT-INFO: relaxing branches for compact code model (<128MB)\n";
@@ -967,6 +1270,12 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun,
SkipPredicate, "RelaxLocalBranches");
+ if (!opts::ExperimentalRelaxation)
+ return Error::success();
+
+ BC.outs() << "BOLT-INFO: starting experimental relaxation pass\n";
+ relaxCalls(BC);
+
return Error::success();
}
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index 58d24e15cde01..4a40c12ba6e89 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -528,6 +528,9 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
if (BC.HasRelocations)
Manager.registerPass(std::make_unique<PatchEntries>());
+ // Assign each function an output section.
+ Manager.registerPass(std::make_unique<AssignSections>());
+
if (BC.isAArch64()) {
Manager.registerPass(
std::make_unique<AArch64RelaxationPass>(PrintAArch64Relaxation));
@@ -555,9 +558,6 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
Manager.registerPass(
std::make_unique<RetpolineInsertion>(PrintRetpolineInsertion));
- // Assign each function an output section.
- Manager.registerPass(std::make_unique<AssignSections>());
-
// This pass turns tail calls into jumps which makes them invisible to
// function reordering. It's unsafe to use any CFG or instruction analysis
// after this point.
diff --git a/bolt/test/AArch64/relax-exp.s b/bolt/test/AArch64/relax-exp.s
new file mode 100644
index 0000000000000..f6394b7a39df7
--- /dev/null
+++ b/bolt/test/AArch64/relax-exp.s
@@ -0,0 +1,78 @@
+## Check that llvm-bolt handles code size larger than 256MB.
+## Additionally, check veneers: no double veneers in lite mode and proper names
+## for BOLT-introduced veneers.
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=1 --data %t.fdata \
+# RUN: --print-normalized 2>&1 | FileCheck %s --check-prefix=CHECK-BOLT-LITE
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=0 --data %t.fdata \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --hot-functions-at-end --lite=0 \
+# RUN: --data %t.fdata | FileCheck %s --check-prefix=CHECK-BOLT-HOT-END
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+
+## Constant islands at the end of functions foo(), bar(), and _start() make each
+## one of them ~112MB in size. Thus the total code size exceeds 300MB.
+
+ .text
+ .global foo
+ .type foo, %function
+foo:
+ bl _start
+ bl bar
+ ret
+ .space 0x7000000
+ .size foo, .-foo
+
+ .global bar
+ .type bar, %function
+bar:
+ bl foo
+ bl _start
+ ret
+ .space 0x7000000
+ .size bar, .-bar
+
+ .global hot
+ .type hot, %function
+hot:
+# FDATA: 0 [unknown] 0 1 hot 0 0 100
+ bl foo
+ bl bar
+ bl _start
+ ret
+ .size hot, .-hot
+
+## Check that BOLT sees the call to foo, not to its veneer in lite mode.
+# CHECK-BOLT-LITE-LABEL: Binary Function "hot"
+# CHECK-BOLT-LITE: bl
+# CHECK-BOLT-LITE-SAME: {{[[:space:]]foo[[:space:]]}}
+
+# CHECK-BOLT-LITE-NOT: BOLT-INFO: {{.*}} short thunks created
+# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
+
+## Check the number of thunks created in other modes.
+# CHECK-BOLT: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT: BOLT-INFO: 3 long thunks created
+
+# CHECK-BOLT-HOT-END: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 2 long thunks created
+
+## Check that correct veneers are used depending on the target proximity.
+# CHECK-OUTPUT-LABEL: <hot>:
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_foo>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk_bar>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
+
+ .global _start
+ .type _start, %function
+_start:
+ bl foo
+ bl bar
+ bl hot
+ ret
+ .space 0x7000000
+ .size _start, .-_start
>From 1b2adabd5a5ff5e13e29e3c5aeec0bc00709ba26 Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks at fb.com>
Date: Tue, 6 Jan 2026 18:02:53 -0800
Subject: [PATCH 2/3] fixup! [BOLT][AArch64] Add experimental call relaxation
pass
---
bolt/include/bolt/Passes/LongJmp.h | 4 ++--
bolt/lib/Passes/LongJmp.cpp | 5 ++---
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 702f942ab260e..bd62384a01d90 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -77,8 +77,8 @@ class LongJmpPass : public BinaryFunctionPass {
void relaxLocalBranches(BinaryFunction &BF);
/// A group of functions that are located within the longest direct
- /// branch/call instruction distance. Functions withing the cluster do not
- /// require a thunk for calls with the same cluster. The cluster may include
+ /// branch/call instruction distance. Functions within the cluster do not
+ /// require a thunk for calls in the same cluster. The cluster may include
/// a set of thunks for covering calls to functions outside.
struct FunctionCluster {
/// All functions in this cluster.
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 59d94f2618e65..c5c66594e7523 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -961,7 +961,6 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
// Conservatively estimate emitted function size. Assume the worst case
// alignment.
auto estimateFunctionSize = [&](const BinaryFunction &BF) -> uint64_t {
- // Conservative estimation of the aligned function size.
if (!BC.shouldEmit(BF))
return 0;
uint64_t Size = BF.estimateSize();
@@ -1090,7 +1089,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
}
}
- /// Create a thunk with +-128MB span.
+ // Create a thunk with +-128MB span.
size_t NumShortThunks = 0;
auto createShortThunk = [&](const MCSymbol *TargetSymbol) {
++NumShortThunks;
@@ -1103,7 +1102,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
return ThunkBF;
};
- /// Create a thunk with +-4GB span.
+ // Create a thunk with +-4GB span.
size_t NumLongThunks = 0;
auto createLongThunk = [&](const MCSymbol *TargetSymbol) {
++NumLongThunks;
>From 4b3d32f2d670b776f042be78c20103f9558663ee Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks at fb.com>
Date: Wed, 14 Jan 2026 18:05:12 -0800
Subject: [PATCH 3/3] [BOLT] Improve code size estimation for call relaxation
pass
---
bolt/lib/Passes/LongJmp.cpp | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index c5c66594e7523..64cd6918004af 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -963,13 +963,18 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
auto estimateFunctionSize = [&](const BinaryFunction &BF) -> uint64_t {
if (!BC.shouldEmit(BF))
return 0;
- uint64_t Size = BF.estimateSize();
- if (BF.hasValidIndex())
- Size += BF.getAlignment();
+ uint64_t Size = BF.estimateSize() + BF.getMaxAlignmentBytes();
+
+ // Each additional fragment can attribute extra bytes due to its alignment
+ // requirements.
+ for ([[maybe_unused]] const FunctionFragment &FF :
+ BF.getLayout().getSplitFragments())
+ Size += BF.getMaxColdAlignmentBytes();
if (BF.hasIslandsInfo()) {
- Size += BF.getConstantIslandAlignment();
Size += BF.estimateConstantIslandSize();
+ if (BF.getConstantIslandAlignment() > BF.getMinAlignment())
+ Size += BF.getConstantIslandAlignment() - BF.getMinAlignment();
}
return Size;
More information about the llvm-commits
mailing list