[llvm] [BOLT][AArch64] Add experimental unconditional branch relaxation (PR #215825)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 08:28:17 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-bolt
Author: Alexandros Lamprineas (labrinea)
<details>
<summary>Changes</summary>
Add support for unconditional branches whose target is beyond 128MB reach.
Order function fragments by emission order.
Create clusters of function fragments that sum to a size below branch reach.
Relax branches that target a remote cluster by creating thunk chains.
Skip thunk creation for same cluster or adjecent clusters within reach.
Account for space added by injected thunks and padding.
Assume these fit in a cluster.
Assisted-by: Codex
---
Patch is 33.40 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/215825.diff
7 Files Affected:
- (modified) bolt/include/bolt/Core/BinaryContext.h (+15-1)
- (modified) bolt/include/bolt/Passes/LongJmp.h (+4)
- (modified) bolt/lib/Core/BinaryContext.cpp (+51-2)
- (modified) bolt/lib/Passes/BinaryPasses.cpp (+5-3)
- (modified) bolt/lib/Passes/LongJmp.cpp (+326-2)
- (modified) bolt/lib/Rewrite/RewriteInstance.cpp (+4-45)
- (added) bolt/test/AArch64/relax-branches-with-thunk-chain.s (+270)
``````````diff
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 240e5a75d1de5..df2d73dac2869 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -569,12 +569,22 @@ class BinaryContext {
}
/// Return functions meant for the output in a sorted order.
- BinaryFunctionListType &getOutputBinaryFunctions() { return OutputFunctions; }
+ const BinaryFunctionListType &getOutputBinaryFunctions() const {
+ return OutputFunctions;
+ }
+
+ /// Update output function list.
+ void updateOutputBinaryFunctions(BinaryFunctionListType &&Functions) {
+ OutputFunctions.swap(Functions);
+ }
/// Create BOLT-injected function
BinaryFunction *createInjectedBinaryFunction(const std::string &Name,
bool IsSimple = true);
+ /// Create BOLT-injected thunk function with a unique name.
+ BinaryFunction *createThunkBinaryFunction(const std::string &Name);
+
/// Patch the original binary contents at address \p Address with a sequence
/// of instructions from the \p Instructions list. The callee is responsible
/// for checking that the sequence doesn't cross any function or section
@@ -1159,6 +1169,10 @@ class BinaryContext {
return ".text.injected.cold";
}
+ /// Return true if \p A should be emitted before \p B in output code section
+ /// order.
+ bool compareSectionNames(StringRef A, StringRef B) const;
+
ErrorOr<BinarySection &> getGdbIndexSection() const {
return getUniqueSectionByName(".gdb_index");
}
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 4a288d3bde4e3..4a2656aecd8b7 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -80,6 +80,10 @@ class LongJmpPass : public BinaryFunctionPass {
bool relaxLocalBranches(BinaryFunction &BF,
const BranchLivenessInfo *BLI = nullptr);
+ /// Relax direct unconditional branches using function fragment clusters.
+ void relaxUnconditionalBranches(BinaryContext &BC,
+ BinaryFunctionListType &OutputFunctions);
+
/// -- Layout estimation methods --
/// Try to do layout before running the emitter, by looking at BinaryFunctions
/// and MCInsts -- this is an estimation. To be correct for longjmp inserter
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 13d7e4bc1a5d6..7f4958330f601 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -14,6 +14,7 @@
#include "bolt/Core/BinaryEmitter.h"
#include "bolt/Core/BinaryFunction.h"
#include "bolt/Utils/CommandLineOpts.h"
+#include "bolt/Utils/NameResolver.h"
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Twine.h"
@@ -342,6 +343,48 @@ bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {
return false;
}
+bool BinaryContext::compareSectionNames(StringRef A, StringRef B) const {
+ if (A == B)
+ return false;
+
+ // If both A and B have names starting with ".text.cold", then
+ // - if opts::HotFunctionsAtEnd is true, we want order
+ // ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
+ // - if opts::HotFunctionsAtEnd is false, we want order
+ // ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
+ if (A.starts_with(getColdCodeSectionName()) &&
+ B.starts_with(getColdCodeSectionName())) {
+ if (A.size() != B.size())
+ return (opts::HotFunctionsAtEnd) ? (A.size() > B.size())
+ : (A.size() < B.size());
+ return (opts::HotFunctionsAtEnd) ? (A > B) : (A < B);
+ }
+
+ // Place hot text movers before anything else.
+ if (opts::HotText) {
+ if (A == getHotTextMoverSectionName())
+ return true;
+ if (B == getHotTextMoverSectionName())
+ return false;
+ }
+
+ // Depending on opts::HotFunctionsAtEnd, place main and warm sections in
+ // order.
+ if (opts::HotFunctionsAtEnd) {
+ if (B == getMainCodeSectionName())
+ return true;
+ if (A == getMainCodeSectionName())
+ return false;
+ return (B == getWarmCodeSectionName());
+ } else {
+ if (A == getMainCodeSectionName())
+ return true;
+ if (B == getMainCodeSectionName())
+ return false;
+ return (A == getWarmCodeSectionName());
+ }
+}
+
std::unique_ptr<MCObjectWriter>
BinaryContext::createObjectWriter(raw_pwrite_stream &OS) {
return MAB->createObjectWriter(OS);
@@ -2643,12 +2686,18 @@ BinaryContext::createInjectedBinaryFunction(const std::string &Name,
setSymbolToFunctionMap(BF->getSymbol(), BF);
BF->CurrentState = BinaryFunction::State::CFG;
- if (!getOutputBinaryFunctions().empty())
- getOutputBinaryFunctions().push_back(BF);
+ if (!OutputFunctions.empty())
+ OutputFunctions.push_back(BF);
return BF;
}
+BinaryFunction *
+BinaryContext::createThunkBinaryFunction(const std::string &Name) {
+ static NameResolver NR;
+ return createInjectedBinaryFunction(NR.uniquify(Name));
+}
+
BinaryFunction *
BinaryContext::createInstructionPatch(uint64_t Address,
const InstructionListType &Instructions,
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index d24295e5c2c32..dcda2dcf08fe8 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -585,10 +585,10 @@ Error FixupBranches::runOnFunctions(BinaryContext &BC) {
}
Error PopulateOutputFunctions::runOnFunctions(BinaryContext &BC) {
- BinaryFunctionListType &OutputFunctions = BC.getOutputBinaryFunctions();
-
- assert(OutputFunctions.empty() && "Output function list already initialized");
+ assert(BC.getOutputBinaryFunctions().empty() &&
+ "Output function list already initialized");
+ BinaryFunctionListType OutputFunctions;
OutputFunctions.reserve(BC.getBinaryFunctions().size() +
BC.getInjectedBinaryFunctions().size());
llvm::transform(llvm::make_second_range(BC.getBinaryFunctions()),
@@ -616,6 +616,8 @@ Error PopulateOutputFunctions::runOnFunctions(BinaryContext &BC) {
[](const BinaryFunction *A) { return !A->hasValidIndex(); });
}
+ BC.updateOutputBinaryFunctions(std::move(OutputFunctions));
+
return Error::success();
}
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 38ad4ed52f339..97a7460a364c6 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -15,6 +15,7 @@
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
@@ -31,12 +32,21 @@ extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool> GroupStubs("group-stubs",
cl::desc("share stubs across functions"),
cl::init(true), cl::cat(BoltOptCategory));
+
+static cl::opt<bool>
+ ExperimentalRelaxation("relax-exp",
+ cl::desc("run experimental relaxation pass"),
+ cl::init(false), cl::cat(BoltOptCategory));
}
namespace llvm {
namespace bolt {
constexpr unsigned ColdFragAlign = 16;
+/// Maximum size of combined code in a relaxation cluster. It is less than the
+/// 128MiB AArch64 branch range so the cluster plus its thunks should fit in
+/// range.
+constexpr uint64_t MaxClusterSize = 112 * 1024 * 1024;
static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();
@@ -1014,9 +1024,316 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
return true;
}
+static uint64_t estimateFragmentSize(const BinaryFunction &BF,
+ const FunctionFragment &FF) {
+ uint64_t Size = 0;
+ for (const BinaryBasicBlock *BB : FF)
+ Size += BB->estimateSize();
+ // Constant islands are emitted at the end of each emitted fragment.
+ // Split functions duplicate the island in the split fragment.
+ if (BF.hasIslandsInfo()) {
+ Size += BF.estimateConstantIslandSize();
+ if (BF.getConstantIslandAlignment() > BF.getMinAlignment())
+ Size += BF.getConstantIslandAlignment() - BF.getMinAlignment();
+ }
+ Size += FF.isSplitFragment() ? BF.getMaxColdAlignmentBytes()
+ : BF.getMaxAlignmentBytes();
+ return Size;
+}
+
+void LongJmpPass::relaxUnconditionalBranches(
+ BinaryContext &BC, BinaryFunctionListType &OutputFunctions) {
+ struct FragmentCluster {
+ size_t Index;
+ SmallString<32> SectionName;
+ uint64_t Size{0};
+
+ /// The indices of the first and last functions contributing fragments to
+ /// this cluster. Used as insertion points for adding thunks to the output
+ /// function list.
+ size_t FirstFunctionIndex = -1;
+ size_t LastFunctionIndex = -1;
+
+ BinaryFunctionListType ForwardThunkList;
+ BinaryFunctionListType BackwardThunkList;
+
+ /// <Destination BB> -> <Thunk Function>.
+ DenseMap<const BinaryBasicBlock *, BinaryFunction *> ForwardThunks;
+ DenseMap<const BinaryBasicBlock *, BinaryFunction *> BackwardThunks;
+ };
+
+ struct OutputFragment {
+ const FunctionFragment *FF;
+ size_t FunctionIndex;
+ SmallString<32> SectionName;
+ };
+
+ SmallVector<FragmentCluster> FragmentClusters;
+ DenseMap<const BinaryBasicBlock *, size_t> BBToFragmentCluster;
+ DenseMap<const BinaryBasicBlock *, uint64_t> BBToClusterOffset;
+
+ struct CrossClusterBranch {
+ MCInst *Inst;
+ BinaryBasicBlock *SourceBB;
+ BinaryBasicBlock *TargetBB;
+ size_t SourceCluster;
+ size_t TargetCluster;
+ };
+
+ SmallVector<CrossClusterBranch> CrossClusterBranches;
+
+ SmallVector<OutputFragment> OrderedFragments;
+ for (size_t BFIndex = 0, NumFuncs = OutputFunctions.size();
+ BFIndex < NumFuncs; ++BFIndex) {
+ BinaryFunction *BF = OutputFunctions[BFIndex];
+ if (!BC.shouldEmit(*BF) || BF->isPatch())
+ continue;
+
+ for (const FunctionFragment &FF : BF->getLayout().fragments()) {
+ if (FF.empty() && !BF->hasConstantIsland())
+ continue;
+
+ OrderedFragments.push_back(
+ {&FF, BFIndex, BF->getCodeSectionName(FF.getFragmentNum())});
+ }
+ }
+
+ // Model final output layout by grouping function fragments in output section
+ // order. Within each section, fragments remain in OutputFunctions order.
+ llvm::stable_sort(
+ OrderedFragments, [&](const OutputFragment &A, const OutputFragment &B) {
+ return BC.compareSectionNames(A.SectionName, B.SectionName);
+ });
+
+ auto addFragmentToCluster = [&](const OutputFragment &Fragment) {
+ BinaryFunction &BF = *OutputFunctions[Fragment.FunctionIndex];
+ const uint64_t FFSize = estimateFragmentSize(BF, *Fragment.FF);
+
+ if (FragmentClusters.empty() ||
+ FragmentClusters.back().SectionName != Fragment.SectionName ||
+ FragmentClusters.back().Size + FFSize > MaxClusterSize) {
+ FragmentClusters.emplace_back();
+ FragmentCluster &Cluster = FragmentClusters.back();
+ Cluster.Index = FragmentClusters.size() - 1;
+ Cluster.SectionName = Fragment.SectionName;
+ }
+
+ FragmentCluster &Cluster = FragmentClusters.back();
+ if (Cluster.FirstFunctionIndex == size_t(-1))
+ Cluster.FirstFunctionIndex = Fragment.FunctionIndex;
+ Cluster.LastFunctionIndex = Fragment.FunctionIndex;
+
+ uint64_t BBOffset = Cluster.Size;
+ for (const BinaryBasicBlock *BB : *Fragment.FF) {
+ BBToFragmentCluster[BB] = Cluster.Index;
+ BBToClusterOffset[BB] = BBOffset;
+ BBOffset += BB->estimateSize();
+ }
+
+ Cluster.Size += FFSize;
+ };
+
+ for (const OutputFragment &Fragment : OrderedFragments)
+ addFragmentToCluster(Fragment);
+
+ BC.outs() << "BOLT-INFO: built " << FragmentClusters.size()
+ << " function fragment clusters\n";
+
+ auto collectCrossClusterBranches = [&]() {
+ for (BinaryFunction *BF : OutputFunctions) {
+ if (!BC.shouldEmit(*BF) || BF->isPatch() || !BF->isSimple())
+ continue;
+
+ for (BinaryBasicBlock &BB : *BF) {
+ for (MCInst &Inst : BB) {
+ if (!BC.MIB->isUnconditionalBranch(Inst))
+ continue;
+
+ // relaxLocalBranches() should have expanded shorter local CFG
+ // branches to local trampolines, leaving only full-range branches
+ // for cluster-level handling here.
+ assert(BC.MIB->getPCRelEncodingSize(Inst) == LongestJumpBits &&
+ "short local branches should already be relaxed");
+
+ const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
+ assert(TargetSymbol && "direct branch expected to have a target");
+ BinaryBasicBlock *TargetBB = BB.getSuccessor(TargetSymbol);
+ if (!TargetBB)
+ continue;
+
+ const size_t SourceCluster = BBToFragmentCluster[&BB];
+ const size_t TargetCluster = BBToFragmentCluster[TargetBB];
+ if (SourceCluster == TargetCluster)
+ continue;
+
+ const uint64_t SourceOffset = BBToClusterOffset[&BB];
+ const uint64_t TargetOffset = BBToClusterOffset[TargetBB];
+
+ // Adjacent clusters can sometimes be reached directly.
+ // Only collect branches that need a thunk chain.
+ if (SourceCluster + 1 == TargetCluster) {
+ const uint64_t EstimatedDistance =
+ FragmentClusters[SourceCluster].Size - SourceOffset +
+ TargetOffset;
+ if (EstimatedDistance < LongestJumpSpan)
+ continue;
+ }
+ if (TargetCluster + 1 == SourceCluster) {
+ const uint64_t EstimatedDistance =
+ SourceOffset + FragmentClusters[TargetCluster].Size -
+ TargetOffset;
+ if (EstimatedDistance < LongestJumpSpan)
+ continue;
+ }
+
+ CrossClusterBranches.push_back(
+ {&Inst, &BB, TargetBB, SourceCluster, TargetCluster});
+ }
+ }
+ }
+ };
+
+ collectCrossClusterBranches();
+
+ // Create a direct branch thunk.
+ size_t NumBranchThunks = 0;
+ auto createBranchThunk = [&](const MCSymbol *TargetSymbol,
+ const bool IsForward) {
+ std::string ThunkName = IsForward ? "__AArch64BranchForwardThunk_"
+ : "__AArch64BranchBackwardThunk_";
+ ThunkName += std::to_string(NumBranchThunks++);
+
+ BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(ThunkName);
+ MCInst Inst;
+ BC.MIB->createUncondBranch(Inst, TargetSymbol, BC.Ctx.get());
+ ThunkBF->addBasicBlock()->addInstruction(Inst);
+
+ return ThunkBF;
+ };
+
+ auto getOrCreateBranchThunk = [&](FragmentCluster &Cluster,
+ const BinaryBasicBlock *TargetBB,
+ const MCSymbol *NextTarget,
+ const bool IsForward) {
+ auto &Thunks = IsForward ? Cluster.ForwardThunks : Cluster.BackwardThunks;
+ auto It = Thunks.find(TargetBB);
+ if (It != Thunks.end())
+ return It->second;
+
+ BinaryFunction *Thunk = createBranchThunk(NextTarget, IsForward);
+ Thunk->setCodeSectionName(Cluster.SectionName);
+ auto &ThunkList =
+ IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
+ ThunkList.push_back(Thunk);
+ Thunks[TargetBB] = Thunk;
+ return Thunk;
+ };
+
+ auto getOrCreateBranchThunkChain =
+ [&](const CrossClusterBranch &Branch) -> const MCSymbol * {
+ const BinaryBasicBlock &SourceBB = *Branch.SourceBB;
+ const BinaryBasicBlock &TargetBB = *Branch.TargetBB;
+ const size_t SourceCluster = Branch.SourceCluster;
+ const size_t TargetCluster = Branch.TargetCluster;
+ BinaryFunction *FirstThunk = nullptr;
+ const MCSymbol *NextTarget = TargetBB.getLabel();
+ const uint64_t SourceOffset = BBToClusterOffset[&SourceBB];
+
+ if (SourceCluster < TargetCluster) {
+ size_t FirstCluster = SourceCluster;
+ if (SourceCluster + 1 < TargetCluster) {
+ // If the source branch can already reach the thunk in the next
+ // cluster, do not create an extra thunk in the source cluster.
+ const uint64_t EstimatedDistance =
+ FragmentClusters[SourceCluster].Size - SourceOffset +
+ FragmentClusters[SourceCluster + 1].Size;
+ if (EstimatedDistance < LongestJumpSpan)
+ FirstCluster = SourceCluster + 1;
+ }
+
+ for (size_t Cluster = TargetCluster; Cluster > FirstCluster;) {
+ --Cluster;
+ FirstThunk =
+ getOrCreateBranchThunk(FragmentClusters[Cluster], &TargetBB,
+ NextTarget, /*IsForward=*/true);
+ NextTarget = FirstThunk->getSymbol();
+ }
+ } else {
+ size_t LastCluster = SourceCluster;
+ if (TargetCluster + 1 < SourceCluster) {
+ // Backward thunks are inserted before their cluster, so a source near
+ // the start of its cluster can often reach the previous cluster thunk.
+ const uint64_t EstimatedDistance =
+ SourceOffset + FragmentClusters[SourceCluster - 1].Size;
+ if (EstimatedDistance < LongestJumpSpan)
+ LastCluster = SourceCluster - 1;
+ }
+
+ for (size_t Cluster = TargetCluster + 1; Cluster <= LastCluster;
+ ++Cluster) {
+ FirstThunk =
+ getOrCreateBranchThunk(FragmentClusters[Cluster], &TargetBB,
+ NextTarget, /*IsForward=*/false);
+ NextTarget = FirstThunk->getSymbol();
+ }
+ }
+
+ assert(FirstThunk && "expected branch thunk chain");
+ return FirstThunk->getSymbol();
+ };
+
+ for (const CrossClusterBranch &Branch : CrossClusterBranches) {
+ const MCSymbol *Target = getOrCreateBranchThunkChain(Branch);
+ BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
+ }
+
+ if (!CrossClusterBranches.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << CrossClusterBranches.size()
+ << " cross-cluster branches\n";
+
+ if (NumBranchThunks)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
+
+ struct BranchThunkInsertion {
+ size_t Position;
+ bool InsertAfter;
+ BinaryFunctionListType *BranchThunkList;
+ };
+
+ SmallVector<BranchThunkInsertion> BranchThunkInsertions;
+ for (FragmentCluster &Cluster : FragmentClusters) {
+ if (!Cluster.BackwardThunkList.empty())
+ BranchThunkInsertions.push_back({Cluster.FirstFunctionIndex,
+ /*InsertAfter=*/false,
+ &Cluster.BackwardThunkList});
+
+ if (!Cluster.ForwardThunkList.empty())
+ BranchThunkInsertions.push_back({Cluster.LastFunctionIndex + 1,
+ /*InsertAfter=*/true,
+ &Cluster.ForwardThunkList});
+ }
+
+ // Apply insertions from high to low indices so earlier insertions do not
+ // invalidate later positions. At a shared boundary, insert backward thunks
+ // first: repeated insertion at the same index places the later insertion
+ // first in the final list, yielding forward thunks before backward thunks.
+ llvm::sort(BranchThunkInsertions,
+ [](const BranchThunkInsertion &A, const BranchThunkInsertion &B) {
+ if (A.Position != B.Position)
+ return A.Position > B.Position;
+ return !A.InsertAfter && B.InsertAfter;
+ });
+
+ for (BranchThunkInsertion &Insertion : BranchThunkInsertions) {
+ OutputFunctions.insert(
+ std::next(OutputFunctions.begin(), Insertion.Position),
+ Insertion.BranchThunkList->begin(), Insertion.BranchThunkList->end());
+ }
+}
+
Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
- assert((opts::CompactCodeModel ||
+ assert((opts::CompactCodeModel || opts::ExperimentalRelaxation ||
opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) &&
"LongJmp cannot work with functions split in more than two fragments");
@@ -1039,7 +1356,7 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
return It == Br...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/215825
More information about the llvm-commits
mailing list