[llvm] [BOLT][AArch64] Relax calls and branches with fragment clusters (PR #215825)
Alexandros Lamprineas via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 01:50:10 PDT 2026
https://github.com/labrinea updated https://github.com/llvm/llvm-project/pull/215825
>From e989c9aea5e047c1eaf8db273a3033e8ffce6d8d Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sun, 23 Aug 2026 19:20:43 +0000
Subject: [PATCH 01/22] [BOLT][AArch64] Relax calls and branches with fragment
clusters
Build relaxation clusters from emitted function fragments instead of
whole functions, so split hot/cold code is modeled in the same order
it will be emitted. Use those clusters to place short and long call
thunks, and to build forward/backward branch thunk chains for
cross-cluster unconditional branches. This lets the experimental
AArch64 relaxation path handle large split binaries where direct
call/branch relocations can cross the +/-128MiB branch range after
BOLT layout changes.
Tested with stage2-clang-bolt and Chromium using Speedometer-generated
profile data, with lite mode disabled and without assuming PLT entries
are nearby.
---
bolt/include/bolt/Core/BinaryContext.h | 3 +
bolt/include/bolt/Core/BinaryFunction.h | 12 +
bolt/include/bolt/Passes/LongJmp.h | 75 ++-
bolt/lib/Core/BinaryContext.cpp | 102 ++-
bolt/lib/Passes/LongJmp.cpp | 597 +++++++++++-------
bolt/lib/Rewrite/RewriteInstance.cpp | 104 +--
.../AArch64/relax-branches-with-thunk-chain.s | 309 +++++++++
.../AArch64/{relax-exp.s => relax-calls.s} | 12 +-
.../test/AArch64/relax-cross-fragment-calls.s | 216 +++++++
9 files changed, 1070 insertions(+), 360 deletions(-)
create mode 100644 bolt/test/AArch64/relax-branches-with-thunk-chain.s
rename bolt/test/AArch64/{relax-exp.s => relax-calls.s} (86%)
create mode 100644 bolt/test/AArch64/relax-cross-fragment-calls.s
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 92cd853870cae..f0bd997effa3a 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -1199,6 +1199,9 @@ class BinaryContext {
return ".text.injected.cold";
}
+ /// Return true if \p A should be emitted before \p B in code section order.
+ bool compareSectionNames(StringRef A, StringRef B) const;
+
ErrorOr<BinarySection &> getGdbIndexSection() const {
return getUniqueSectionByName(".gdb_index");
}
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 14d7f9b5b5359..3865e87dc25ef 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -394,6 +394,9 @@ class BinaryFunction {
/// True if the function is used for patching code at a fixed address.
bool IsPatch{false};
+ /// True if the function is a synthetic branch/call thunk.
+ bool IsThunk{false};
+
/// True if the original entry point of the function may get called, but the
/// original body cannot be executed and needs to be patched with code that
/// redirects execution to the new function body.
@@ -1504,6 +1507,9 @@ class BinaryFunction {
/// Return true if this function is used for patching existing code.
bool isPatch() const { return IsPatch; }
+ /// Return true if this function is a synthetic branch/call thunk.
+ bool isThunk() const { return IsThunk; }
+
/// Return true if the function requires a patch.
bool needsPatch() const { return NeedsPatch; }
@@ -1941,6 +1947,12 @@ class BinaryFunction {
IsPatch = V;
}
+ /// Indicate that this function is a synthetic branch/call thunk.
+ void setIsThunk(bool V) {
+ assert(isInjected() && "Only injected functions can be used as thunks");
+ IsThunk = V;
+ }
+
/// Mark the function for patching.
void setNeedsPatch(bool V) { NeedsPatch = V; }
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index fa8cf67a95d73..37c7e1452d5bf 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -10,6 +10,8 @@
#define BOLT_PASSES_LONGJMP_H
#include "bolt/Passes/BinaryPasses.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
namespace llvm {
namespace bolt {
@@ -80,46 +82,71 @@ class LongJmpPass : public BinaryFunctionPass {
bool relaxLocalBranches(BinaryFunction &BF,
const BranchLivenessInfo *BLI = nullptr);
- /// A group of functions that are located within the longest direct
- /// branch/call instruction distance. Functions within the cluster do not
- /// require a thunk for calls in the same cluster. The cluster may include
- /// a set of thunks for covering calls to functions outside.
- struct FunctionCluster {
- /// All functions in this cluster.
- DenseSet<BinaryFunction *> Functions;
-
- /// Symbols corresponding to entry points of functions that this cluster
- /// calls. Note that it excludes all functions in the cluster itself.
- DenseSet<const MCSymbol *> Callees;
+ /// A group of function fragments that are located within the longest direct
+ /// branch/call instruction distance. Jumps within the cluster do not require
+ /// a thunk. The cluster may include thunks for jumps to targets outside.
+ struct FragmentCluster {
+ /// Output code section containing fragments in this cluster.
+ SmallString<32> SectionName;
/// Estimated size of the cluster in bytes.
uint64_t Size{0};
- /// The index of the last function in the cluster. Used as an insertion
- /// point for adding thunks to the output function list.
- size_t LastFunctionIndex = -1;
+ /// Number of function fragments in the cluster.
+ size_t NumFragments{0};
- /// When placing hot code at the end of the binary, track the first function
- /// for insertion purposes.
+ /// The indices of the first and last functions contributing fragments to
+ /// this cluster. Used as insertion points for adding thunks to the output
+ /// function list.
size_t FirstFunctionIndex = -1;
+ size_t LastFunctionIndex = -1;
- /// Thunks located at the end of this cluster.
- BinaryFunctionListType ThunkList;
+ /// Thunks located after this cluster.
+ BinaryFunctionListType ForwardThunkList;
- /// Thunks used by this cluster. Some could be in a ThunkList of the
- /// preceding cluster.
+ /// Thunks located before this cluster.
+ BinaryFunctionListType BackwardThunkList;
+
+ /// Call thunks used by this cluster.
///
/// <Function Symbol> -> <Thunk Function>.
- DenseMap<const MCSymbol *, BinaryFunction *> Thunks;
+ DenseMap<const MCSymbol *, BinaryFunction *> CallThunks;
+
+ /// <Destination Symbol> -> <Thunk Function>.
+ DenseMap<const MCSymbol *, BinaryFunction *> ForwardBranchThunks;
+ DenseMap<const MCSymbol *, BinaryFunction *> BackwardBranchThunks;
};
/// Maximum size of combined regular functions in the cluster. Note that it's
/// less than 128MB, because the size of the cluster plus its thunks should be
/// less than 128MB.
- static constexpr uint64_t MaxClusterSize = 125 * 1024 * 1024;
+ static constexpr uint64_t MaxClusterSize = 120 * 1024 * 1024;
+
+ struct FragmentClusterLayout {
+ SmallVector<FragmentCluster, 4> Clusters;
+ DenseMap<const BinaryBasicBlock *, unsigned> BBToCluster;
+ DenseMap<const MCSymbol *, unsigned> SymToCluster;
+ };
+
+ FragmentClusterLayout
+ buildClusterLayout(BinaryContext &BC,
+ const BinaryFunctionListType &OutputFunctions);
+
+ /// Relax calls using function fragment clusters.
+ void relaxCalls(BinaryContext &BC, BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout);
+
+ /// Relax direct unconditional branches using function fragment clusters.
+ void relaxUnconditionalBranches(BinaryContext &BC,
+ BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout);
+
+ /// Insert all thunks owned by the fragment cluster layout.
+ void insertClusterThunks(BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout);
- /// Relax calls using function cluster approach.
- void relaxCalls(BinaryContext &BC);
+ /// Relax calls and direct unconditional branches using one cluster layout.
+ void relaxWithClusters(BinaryContext &BC);
/// -- Layout estimation methods --
/// Try to do layout before running the emitter, by looking at BinaryFunctions
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index d911f8191f791..94d8770dae493 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -54,6 +54,7 @@ namespace opts {
extern cl::opt<bool> LargeCodeModel;
extern cl::opt<bool> UpdateDebugSections;
+extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool>
NoHugePages("no-huge-pages",
@@ -353,6 +354,103 @@ bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {
return false;
}
+namespace {
+
+/// Defines the strict weak ordering for BOLT-produced code sections.
+class CodeSectionOrder {
+public:
+ CodeSectionOrder(StringRef ColdSectionName, StringRef HotTextMoverSectionName,
+ StringRef MainSectionName, StringRef WarmSectionName,
+ bool HotText, bool HotFunctionsAtEnd)
+ : ColdSectionName(ColdSectionName),
+ HotTextMoverSectionName(HotTextMoverSectionName),
+ MainSectionName(MainSectionName), WarmSectionName(WarmSectionName),
+ HotText(HotText), HotFunctionsAtEnd(HotFunctionsAtEnd) {}
+
+ bool operator()(StringRef AName, StringRef BName) const {
+ const SectionKind AKind = getKind(AName);
+ const SectionKind BKind = getKind(BName);
+ const unsigned ARank = getRank(AKind);
+ const unsigned BRank = getRank(BKind);
+ if (ARank != BRank)
+ return ARank < BRank;
+
+ if (AKind == SectionKind::Cold) {
+ if (AName.size() != BName.size())
+ return HotFunctionsAtEnd ? AName.size() > BName.size()
+ : AName.size() < BName.size();
+ if (AName != BName)
+ return HotFunctionsAtEnd ? AName > BName : AName < BName;
+ }
+
+ return false;
+ }
+
+private:
+ enum class SectionKind { Mover, Main, Warm, Cold, Other };
+
+ SectionKind getKind(StringRef Name) const {
+ if (HotText && Name == HotTextMoverSectionName)
+ return SectionKind::Mover;
+ if (Name == MainSectionName)
+ return SectionKind::Main;
+ if (Name == WarmSectionName)
+ return SectionKind::Warm;
+ if (Name.starts_with(ColdSectionName))
+ return SectionKind::Cold;
+ return SectionKind::Other;
+ }
+
+ unsigned getRank(SectionKind Kind) const {
+ if (Kind == SectionKind::Mover)
+ return 0;
+ if (HotFunctionsAtEnd) {
+ switch (Kind) {
+ case SectionKind::Other:
+ return 1;
+ case SectionKind::Cold:
+ return 2;
+ case SectionKind::Warm:
+ return 3;
+ case SectionKind::Main:
+ return 4;
+ case SectionKind::Mover:
+ llvm_unreachable("handled above");
+ }
+ }
+ switch (Kind) {
+ case SectionKind::Main:
+ return 1;
+ case SectionKind::Warm:
+ return 2;
+ case SectionKind::Cold:
+ return 3;
+ case SectionKind::Other:
+ return 4;
+ case SectionKind::Mover:
+ llvm_unreachable("handled above");
+ }
+ llvm_unreachable("unknown section kind");
+ }
+
+ StringRef ColdSectionName;
+ StringRef HotTextMoverSectionName;
+ StringRef MainSectionName;
+ StringRef WarmSectionName;
+ bool HotText;
+ bool HotFunctionsAtEnd;
+};
+
+} // namespace
+
+bool BinaryContext::compareSectionNames(StringRef A, StringRef B) const {
+ const CodeSectionOrder CompareSections(
+ getColdCodeSectionName(), getHotTextMoverSectionName(),
+ getMainCodeSectionName(), getWarmCodeSectionName(), opts::HotText,
+ opts::HotFunctionsAtEnd);
+ return CompareSections(A, B);
+}
+
std::unique_ptr<MCObjectWriter>
BinaryContext::createObjectWriter(raw_pwrite_stream &OS) {
return MAB->createObjectWriter(OS);
@@ -2816,7 +2914,9 @@ BinaryContext::createInstructionPatch(uint64_t Address,
BinaryFunction *
BinaryContext::createThunkBinaryFunction(const std::string &Name) {
static NameResolver NR;
- return createInjectedBinaryFunction(NR.uniquify(Name));
+ BinaryFunction *BF = createInjectedBinaryFunction(NR.uniquify(Name));
+ BF->setIsThunk(true);
+ return BF;
}
std::pair<size_t, size_t>
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 30ff9dc4ddc71..ccdb2ef37f77f 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1023,297 +1023,434 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
return true;
}
-void LongJmpPass::relaxCalls(BinaryContext &BC) {
- // Operate on a copy of binary functions. We are going to manually insert new
- // thunks and update the list.
- BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
+static uint64_t estimateFragmentSize(const BinaryFunction &BF,
+ const FunctionFragment &FF) {
+ uint64_t Size = 0;
+ for (const BinaryBasicBlock *BB : FF)
+ Size += BB->estimateSize();
+
+ if (BF.hasIslandsInfo()) {
+ Size += BF.estimateConstantIslandSize();
+ if (BF.getConstantIslandAlignment() > BF.getMinAlignment())
+ Size += BF.getConstantIslandAlignment() - BF.getMinAlignment();
+ }
- // Conservatively estimate emitted function size. Assume the worst case
- // alignment.
- auto estimateFunctionSize = [&](const BinaryFunction &BF) -> uint64_t {
- if (!BC.shouldEmit(BF))
- return 0;
- uint64_t Size = BF.estimateSize() + BF.getMaxAlignmentBytes();
-
- // Each additional fragment can attribute extra bytes due to its alignment
- // requirements.
- for ([[maybe_unused]] const FunctionFragment &FF :
- BF.getLayout().getSplitFragments())
- Size += BF.getMaxColdAlignmentBytes();
-
- if (BF.hasIslandsInfo()) {
- Size += BF.estimateConstantIslandSize();
- if (BF.getConstantIslandAlignment() > BF.getMinAlignment())
- Size += BF.getConstantIslandAlignment() - BF.getMinAlignment();
- }
+ Size += FF.isSplitFragment() ? BF.getMaxColdAlignmentBytes()
+ : BF.getMaxAlignmentBytes();
+ return Size;
+}
- return Size;
+LongJmpPass::FragmentClusterLayout
+LongJmpPass::buildClusterLayout(BinaryContext &BC,
+ const BinaryFunctionListType &OutputFunctions) {
+ struct OutputFragment {
+ const FunctionFragment *FF;
+ size_t FunctionIndex;
+ SmallString<32> SectionName;
};
- // Map every function to its direct callees. Note that this is different from
- // the regular call graph as here we completely ignore indirect calls.
+ FragmentClusterLayout Layout;
+ SmallVector<OutputFragment> OrderedFragments;
uint64_t EstimatedSize = 0;
- DenseMap<BinaryFunction *, std::set<const MCSymbol *>> CallMap;
- for (BinaryFunction *BF : OutputFunctions) {
+ for (size_t I = 0; I < OutputFunctions.size(); ++I) {
+ BinaryFunction *BF = OutputFunctions[I];
if (!BC.shouldEmit(*BF) || BF->isPatch())
continue;
- EstimatedSize += estimateFunctionSize(*BF);
-
- for (const BinaryBasicBlock &BB : *BF) {
- for (const MCInst &Inst : BB) {
- if (!BC.MIB->isCall(Inst) || BC.MIB->isIndirectCall(Inst) ||
- BC.MIB->isIndirectBranch(Inst))
- continue;
- const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
- assert(TargetSymbol);
-
- // Ignore internal calls that use basic block labels as a destination.
- if (!BC.getFunctionForSymbol(TargetSymbol))
- continue;
+ for (const FunctionFragment &FF : BF->getLayout().fragments()) {
+ if (FF.empty() && !BF->hasConstantIsland())
+ continue;
- CallMap[BF].insert(TargetSymbol);
- }
+ OrderedFragments.push_back(
+ {&FF, I, BF->getCodeSectionName(FF.getFragmentNum())});
}
}
- LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << EstimatedSize
- << '\n');
-
- // Build clusters in the order the functions will appear in the output.
- std::vector<FunctionCluster> Clusters;
- for (size_t Index = 0, NumFuncs = OutputFunctions.size(); Index < NumFuncs;
- ++Index) {
- const size_t BFIndex =
- opts::HotFunctionsAtEnd ? NumFuncs - Index - 1 : Index;
- BinaryFunction *BF = OutputFunctions[BFIndex];
- if (!BC.shouldEmit(*BF) || BF->isPatch())
- continue;
+ // Model final output layout by grouping function fragments in output section
+ // order. Within each section, fragments remain in OutputFunctions order.
+ llvm::stable_sort(
+ OrderedFragments, [&](const OutputFragment &A, const OutputFragment &B) {
+ return BC.compareSectionNames(A.SectionName, B.SectionName);
+ });
- const uint64_t BFSize = estimateFunctionSize(*BF);
- if (Clusters.empty() || Clusters.back().Size + BFSize > MaxClusterSize) {
- Clusters.emplace_back(FunctionCluster());
- Clusters.back().FirstFunctionIndex = BFIndex;
+ auto addFragmentToCluster = [&](const OutputFragment &Fragment) {
+ BinaryFunction &BF = *OutputFunctions[Fragment.FunctionIndex];
+ const FunctionFragment &FF = *Fragment.FF;
+ const uint64_t FFSize = estimateFragmentSize(BF, FF);
+
+ if (Layout.Clusters.empty() ||
+ Layout.Clusters.back().SectionName != Fragment.SectionName ||
+ Layout.Clusters.back().Size + FFSize > MaxClusterSize) {
+ Layout.Clusters.emplace_back(FragmentCluster());
+ FragmentCluster &FC = Layout.Clusters.back();
+ FC.SectionName = Fragment.SectionName;
+ FC.FirstFunctionIndex = Fragment.FunctionIndex;
}
- FunctionCluster &FC = Clusters.back();
- FC.Functions.insert(BF);
-
- // When a function is added to the cluster, we have to remove all of its
- // symbols from the cluster callee list. These include alternative symbols
- // (e.g. after ICF) and secondary entry point symbols.
- for (const MCSymbol *Symbol : BF->getSymbols()) {
- auto It = FC.Callees.find(Symbol);
- if (It != FC.Callees.end())
- FC.Callees.erase(It);
- }
- BF->forEachEntryPoint(
- [&FC](uint64_t Offset, const MCSymbol *EntrySymbol) -> bool {
- auto It = FC.Callees.find(EntrySymbol);
- if (It != FC.Callees.end())
- FC.Callees.erase(It);
- return true;
- });
-
- // Update cluster callee list with added function callees.
- for (const MCSymbol *CalleeSymbol : CallMap[BF]) {
- BinaryFunction *Callee = BC.getFunctionForSymbol(CalleeSymbol);
- if (!FC.Functions.count(Callee)) {
- FC.Callees.insert(CalleeSymbol);
- }
+ FragmentCluster &FC = Layout.Clusters.back();
+ FC.LastFunctionIndex = Fragment.FunctionIndex;
+ ++FC.NumFragments;
+ EstimatedSize += FFSize;
+ const unsigned ClusterNum = Layout.Clusters.size() - 1;
+
+ // Map primary entry points.
+ if (FF.isMainFragment())
+ for (const MCSymbol *Symbol : BF.getSymbols())
+ Layout.SymToCluster[Symbol] = ClusterNum;
+
+ for (const BinaryBasicBlock *BB : FF) {
+ Layout.BBToCluster[BB] = ClusterNum;
+ if (const MCSymbol *Label = BB->getLabel())
+ Layout.SymToCluster[Label] = ClusterNum;
+
+ // Map secondary entry points.
+ if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
+ Layout.SymToCluster[EntrySymbol] = ClusterNum;
}
- FC.Size += BFSize;
- FC.LastFunctionIndex = BFIndex;
- }
+ FC.Size += FFSize;
+ };
- if (opts::HotFunctionsAtEnd) {
- std::reverse(Clusters.begin(), Clusters.end());
- llvm::for_each(Clusters, [](FunctionCluster &FC) {
- std::swap(FC.LastFunctionIndex, FC.FirstFunctionIndex);
- });
- }
+ for (const OutputFragment &Fragment : OrderedFragments)
+ addFragmentToCluster(Fragment);
- if (Clusters.empty())
- return;
+ if (Layout.Clusters.empty())
+ return Layout;
+
+ LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << EstimatedSize
+ << '\n');
// Print cluster stats.
- BC.outs() << "BOLT-INFO: built " << Clusters.size()
- << " function cluster(s)\n";
- uint64_t ClusterIndex = 0;
- for (const FunctionCluster &FC : Clusters) {
- BC.outs() << "BOLT-INFO: cluster: " << ClusterIndex++ << '\n'
- << "BOLT-INFO: " << FC.Functions.size() << " function(s)\n"
- << "BOLT-INFO: " << FC.Callees.size() << " callee(s)\n"
+ BC.outs() << "BOLT-INFO: built " << Layout.Clusters.size()
+ << " function fragment cluster(s)\n";
+ for (size_t I = 0; I < Layout.Clusters.size(); ++I) {
+ const FragmentCluster &FC = Layout.Clusters[I];
+ BC.outs() << "BOLT-INFO: cluster: " << I << '\n'
+ << "BOLT-INFO: " << FC.NumFragments << " fragment(s)\n"
<< "BOLT-INFO: " << FC.Size << " estimated bytes\n";
}
if (opts::RelaxPLT) {
// Populate one of the clusters with PLT functions based on the proximity of
// the PLT section to avoid unneeded thunk redirection.
- const size_t PLTClusterNum = opts::UseOldText ? Clusters.size() - 1 : 0;
- auto &PLTCluster = Clusters[PLTClusterNum];
- for (BinaryFunction &BF :
- llvm::make_second_range(BC.getBinaryFunctions())) {
+ const unsigned PLTCluster =
+ opts::UseOldText ? Layout.Clusters.size() - 1 : 0;
+ for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions()))
if (BF.isPLTFunction()) {
- PLTCluster.Functions.insert(&BF);
- auto It = PLTCluster.Callees.find(BF.getSymbol());
- if (It != PLTCluster.Callees.end())
- PLTCluster.Callees.erase(It);
+ for (const MCSymbol *Symbol : BF.getSymbols())
+ Layout.SymToCluster[Symbol] = PLTCluster;
+ BF.forEachEntryPoint(
+ [&](uint64_t, const MCSymbol *EntrySymbol) -> bool {
+ Layout.SymToCluster[EntrySymbol] = PLTCluster;
+ return true;
+ });
}
- }
}
- // Create a thunk with +-128MB span.
- size_t NumShortThunks = 0;
- auto createShortThunk = [&](const MCSymbol *TargetSymbol) {
- ++NumShortThunks;
- BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(
- "__AArch64Thunk_" + TargetSymbol->getName().str());
- MCInst Inst;
- BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
- ThunkBF->addBasicBlock()->addInstruction(Inst);
+ return Layout;
+}
- return ThunkBF;
+void LongJmpPass::relaxCalls(BinaryContext &BC,
+ BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout) {
+ auto &Clusters = Layout.Clusters;
+ auto &BBToCluster = Layout.BBToCluster;
+ auto &SymToCluster = Layout.SymToCluster;
+
+ struct CrossClusterCall {
+ MCInst *Inst;
+ const MCSymbol *TargetSymbol;
+ unsigned SourceCluster;
+ unsigned TargetCluster;
};
- // Create a thunk with +-4GB span.
- size_t NumLongThunks = 0;
- auto createLongThunk = [&](const MCSymbol *TargetSymbol) {
- ++NumLongThunks;
- BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(
- "__AArch64ADRPThunk_" + TargetSymbol->getName().str());
- InstructionListType Instructions;
- BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
- ThunkBF->addBasicBlock()->addInstructions(Instructions);
+ SmallVector<CrossClusterCall> CrossClusterCalls;
+ for (BinaryFunction *BF : OutputFunctions) {
+ if (!BC.shouldEmit(*BF) || BF->isPatch())
+ continue;
- return ThunkBF;
- };
+ for (BinaryBasicBlock &BB : *BF) {
+ auto SourceIt = BBToCluster.find(&BB);
+ if (SourceIt == BBToCluster.end())
+ continue;
+ const unsigned SourceCluster = SourceIt->second;
- for (unsigned ClusterNum = 0; ClusterNum < Clusters.size(); ++ClusterNum) {
- FunctionCluster &FC = Clusters[ClusterNum];
- SmallVector<const MCSymbol *, 16> Callees(FC.Callees.begin(),
- FC.Callees.end());
-
- // Generate thunks in deterministic order.
- llvm::sort(Callees, [&BC](const MCSymbol *A, const MCSymbol *B) {
- uint64_t EntryA;
- uint64_t EntryB;
- BinaryFunction *BFA = BC.getFunctionForSymbol(A, &EntryA);
- BinaryFunction *BFB = BC.getFunctionForSymbol(B, &EntryB);
- if (BFA == BFB) {
- if (EntryA != EntryB)
- return EntryA < EntryB;
-
- // Use lexicographical order for ICF'ed symbols.
- return A->getName() < B->getName();
- }
- return compareBinaryFunctionByIndex(BFA, BFB);
- });
+ for (MCInst &Inst : BB) {
+ if (!BC.MIB->isCall(Inst) && !BC.MIB->isUnconditionalBranch(Inst))
+ continue;
- // Return index of adjacent cluster containing the function.
- auto getAdjClusterWithFunction =
- [&](const BinaryFunction *BF) -> std::optional<unsigned> {
- if (ClusterNum > 0 && Clusters[ClusterNum - 1].Functions.count(BF))
- return ClusterNum - 1;
- if (ClusterNum + 1 < Clusters.size() &&
- Clusters[ClusterNum + 1].Functions.count(BF))
- return ClusterNum + 1;
- return std::nullopt;
- };
+ const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
+ if (!TargetSymbol)
+ continue;
- const FunctionCluster *PrevCluster =
- ClusterNum ? &Clusters[ClusterNum - 1] : nullptr;
+ auto It = SymToCluster.find(TargetSymbol);
+ const bool Found = It != SymToCluster.end();
+ // Known plain branches use B-only thunk chains.
+ // Unknown address-only branches may use tail-call thunks.
+ if (BC.MIB->isUnconditionalBranch(Inst) && Found)
+ continue;
- // Create short thunks for callees in adjacent clusters and long thunks
- // for callees outside.
- for (const MCSymbol *Callee : Callees) {
- if (FC.Thunks.count(Callee))
- continue;
+ if (!BC.getFunctionForSymbol(TargetSymbol) &&
+ !BC.getSymbolValue(*TargetSymbol))
+ continue;
- BinaryFunction *Thunk = 0;
- std::optional<unsigned> AdjCluster =
- getAdjClusterWithFunction(BC.getFunctionForSymbol(Callee));
- if (AdjCluster) {
- Thunk = createShortThunk(Callee);
- } else {
- // Previous cluster may already have a long thunk that can be reused.
- if (PrevCluster) {
- auto It = PrevCluster->Thunks.find(Callee);
- // Reuse only if previous cluster hosts this thunk.
- if (It != PrevCluster->Thunks.end() &&
- llvm::is_contained(PrevCluster->ThunkList, It->second)) {
- FC.Thunks[Callee] = It->second;
- continue;
- }
- }
- Thunk = createLongThunk(Callee);
- }
+ // If not found TargetCluster becomes UINT_MAX.
+ unsigned TargetCluster = Found ? It->second : -1;
+ if (TargetCluster == SourceCluster)
+ continue;
- // The cluster that will host this thunk. If the current cluster is the
- // last one, try to use the previous one. Matters when we want to have hot
- // functions at higher addresses under HotFunctionsAtEnd.
- FunctionCluster *ThunkCluster = &Clusters[ClusterNum];
- if ((AdjCluster && *AdjCluster == ClusterNum - 1) ||
- (ClusterNum && ClusterNum == Clusters.size() - 1))
- ThunkCluster = &Clusters[ClusterNum - 1];
- ThunkCluster->ThunkList.push_back(Thunk);
-
- // Register thunks for all symbols associated with the function.
- uint64_t EntryID = 0;
- const BinaryFunction *BF = BC.getFunctionForSymbol(Callee, &EntryID);
- if (EntryID != 0) {
- FC.Thunks[Callee] = Thunk;
- } else {
- for (const MCSymbol *Symbol : BF->getSymbols()) {
- FC.Thunks[Symbol] = Thunk;
- }
+ CrossClusterCalls.push_back(
+ {&Inst, TargetSymbol, SourceCluster, TargetCluster});
}
}
}
+ size_t NumShortThunks = 0;
+ size_t NumLongThunks = 0;
+ auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort) {
+ BinaryFunction *Thunk = nullptr;
+ if (IsShort) {
+ ++NumShortThunks;
+ Thunk = BC.createThunkBinaryFunction("__AArch64Thunk_" +
+ TargetSymbol->getName().str());
+ MCInst Inst;
+ BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
+ Thunk->addBasicBlock()->addInstruction(Inst);
+ } else {
+ ++NumLongThunks;
+ Thunk = BC.createThunkBinaryFunction("__AArch64ADRPThunk_" +
+ TargetSymbol->getName().str());
+ InstructionListType Instructions;
+ BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
+ Thunk->addBasicBlock()->addInstructions(Instructions);
+ }
+ return Thunk;
+ };
+
+ auto registerCallThunk = [&](FragmentCluster &FC, const MCSymbol *Callee,
+ BinaryFunction *Thunk) {
+ uint64_t EntryID = 0;
+ const BinaryFunction *BF = BC.getFunctionForSymbol(Callee, &EntryID);
+ const bool IsPrimaryEntry = EntryID == 0;
+ if (BF && IsPrimaryEntry)
+ for (const MCSymbol *Symbol : BF->getSymbols())
+ FC.CallThunks[Symbol] = Thunk;
+ else
+ FC.CallThunks[Callee] = Thunk;
+ };
+
+ auto getOrCreateCallThunk = [&](const MCSymbol *TargetSymbol,
+ unsigned SourceCluster, bool IsShort,
+ bool IsForward) {
+ FragmentCluster &FC = Clusters[SourceCluster];
+ if (auto It = FC.CallThunks.find(TargetSymbol); It != FC.CallThunks.end())
+ return It->second;
+
+ BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort);
+
+ FragmentCluster &ThunkCluster = Clusters[SourceCluster];
+ Thunk->setCodeSectionName(ThunkCluster.SectionName);
+ BinaryFunctionListType &ThunkList = IsForward
+ ? ThunkCluster.ForwardThunkList
+ : ThunkCluster.BackwardThunkList;
+ ThunkList.push_back(Thunk);
+
+ // Register thunks for all symbols associated with the function.
+ registerCallThunk(FC, TargetSymbol, Thunk);
+ return Thunk;
+ };
+
+ auto areAdjacent = [](unsigned A, unsigned B) {
+ return A > B ? A - B == 1 : B - A == 1;
+ };
+
+ for (CrossClusterCall &Call : CrossClusterCalls) {
+ const bool IsForward = Call.SourceCluster < Call.TargetCluster;
+ const bool Adjacent = areAdjacent(Call.SourceCluster, Call.TargetCluster);
+
+ BinaryFunction *Thunk = getOrCreateCallThunk(
+ Call.TargetSymbol, Call.SourceCluster, Adjacent, IsForward);
+ BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
+ }
+
+ if (!CrossClusterCalls.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << CrossClusterCalls.size()
+ << " calls with thunks\n";
+
if (NumShortThunks)
BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
if (NumLongThunks)
BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
+}
- // Replace callees with thunks.
- for (FunctionCluster &FC : Clusters) {
- for (BinaryFunction *BF : FC.Functions) {
- if (!CallMap.count(BF))
+void LongJmpPass::relaxUnconditionalBranches(
+ BinaryContext &BC, BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout) {
+ auto &Clusters = Layout.Clusters;
+ auto &BBToCluster = Layout.BBToCluster;
+ auto &SymToCluster = Layout.SymToCluster;
+
+ struct CrossClusterBranch {
+ MCInst *Inst;
+ const MCSymbol *TargetSymbol;
+ unsigned SourceCluster;
+ unsigned TargetCluster;
+ };
+
+ SmallVector<CrossClusterBranch> CrossClusterBranches;
+ for (BinaryFunction *BF : OutputFunctions) {
+ if (!BC.shouldEmit(*BF) || BF->isPatch())
+ continue;
+
+ for (BinaryBasicBlock &BB : *BF) {
+ auto SourceIt = BBToCluster.find(&BB);
+ if (SourceIt == BBToCluster.end())
continue;
+ const unsigned SourceCluster = SourceIt->second;
- for (BinaryBasicBlock &BB : *BF) {
- for (MCInst &Inst : BB) {
- if (!BC.MIB->isCall(Inst) || BC.MIB->isIndirectCall(Inst) ||
- BC.MIB->isIndirectBranch(Inst))
- continue;
- const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
- assert(TargetSymbol);
+ for (MCInst &Inst : BB) {
+ if (!BC.MIB->isUnconditionalBranch(Inst))
+ continue;
- auto It = FC.Thunks.find(TargetSymbol);
- if (It != FC.Thunks.end())
- BC.MIB->replaceBranchTarget(Inst, It->second->getSymbol(),
- BC.Ctx.get());
- }
+ const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
+ if (!TargetSymbol)
+ continue;
+
+ auto TargetIt = SymToCluster.find(TargetSymbol);
+ if (TargetIt == SymToCluster.end())
+ continue;
+
+ const unsigned TargetCluster = TargetIt->second;
+ if (SourceCluster == TargetCluster)
+ continue;
+
+ CrossClusterBranches.push_back(
+ {&Inst, TargetSymbol, SourceCluster, TargetCluster});
}
}
}
- // Add thunks to the function list and assign a section name matching the
- // function they follow.
- for (const FunctionCluster &FC : llvm::reverse(Clusters)) {
- std::string SectionName =
- OutputFunctions[FC.LastFunctionIndex]->getCodeSectionName().str().str();
- for (BinaryFunction *Thunk : FC.ThunkList) {
- Thunk->setCodeSectionName(SectionName);
+ size_t NumBranchThunks = 0;
+ auto createBranchThunk = [&](const MCSymbol *TargetSymbol,
+ const bool IsForward) {
+ std::string ThunkName = IsForward ? "__AArch64BranchForwardThunk_"
+ : "__AArch64BranchBackwardThunk_";
+ ThunkName += std::to_string(NumBranchThunks++);
+
+ BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(ThunkName);
+ MCInst Inst;
+ BC.MIB->createUncondBranch(Inst, TargetSymbol, BC.Ctx.get());
+ ThunkBF->addBasicBlock()->addInstruction(Inst);
+ return ThunkBF;
+ };
+
+ auto getOrCreateBranchThunk =
+ [&](FragmentCluster &Cluster, const MCSymbol *TargetSymbol,
+ const MCSymbol *NextTarget, const bool IsForward) {
+ auto &Thunks = IsForward ? Cluster.ForwardBranchThunks
+ : Cluster.BackwardBranchThunks;
+ auto It = Thunks.find(TargetSymbol);
+ if (It != Thunks.end())
+ return It->second;
+
+ BinaryFunction *Thunk = createBranchThunk(NextTarget, IsForward);
+ Thunk->setCodeSectionName(Cluster.SectionName);
+ auto &ThunkList =
+ IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
+ ThunkList.push_back(Thunk);
+ Thunks[TargetSymbol] = Thunk;
+ return Thunk;
+ };
+
+ auto getOrCreateBranchThunkChain =
+ [&](const CrossClusterBranch &Branch) -> const MCSymbol * {
+ const unsigned SourceCluster = Branch.SourceCluster;
+ const unsigned TargetCluster = Branch.TargetCluster;
+ BinaryFunction *FirstThunk = nullptr;
+ const MCSymbol *NextTarget = Branch.TargetSymbol;
+
+ if (SourceCluster < TargetCluster) {
+ for (unsigned Cluster = TargetCluster; Cluster > SourceCluster;) {
+ --Cluster;
+ FirstThunk =
+ getOrCreateBranchThunk(Clusters[Cluster], Branch.TargetSymbol,
+ NextTarget, /*IsForward=*/true);
+ NextTarget = FirstThunk->getSymbol();
+ }
+ } else {
+ for (unsigned Cluster = TargetCluster + 1; Cluster <= SourceCluster;
+ ++Cluster) {
+ FirstThunk =
+ getOrCreateBranchThunk(Clusters[Cluster], Branch.TargetSymbol,
+ NextTarget, /*IsForward=*/false);
+ NextTarget = FirstThunk->getSymbol();
+ }
}
+ assert(FirstThunk && "expected branch thunk chain");
+ return FirstThunk->getSymbol();
+ };
+
+ for (const CrossClusterBranch &Branch : CrossClusterBranches) {
+ const MCSymbol *Target = getOrCreateBranchThunkChain(Branch);
+ BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
+ }
+
+ if (!CrossClusterBranches.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << CrossClusterBranches.size()
+ << " cross-cluster branches\n";
+
+ if (NumBranchThunks)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
+}
+
+void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
+ FragmentClusterLayout &Layout) {
+ struct ThunkInsertion {
+ size_t Position;
+ bool IsForward;
+ BinaryFunctionListType *ThunkList;
+ };
+
+ SmallVector<ThunkInsertion> Insertions;
+ for (FragmentCluster &Cluster : Layout.Clusters) {
+ if (!Cluster.BackwardThunkList.empty())
+ Insertions.push_back({Cluster.FirstFunctionIndex, /*IsForward=*/false,
+ &Cluster.BackwardThunkList});
+
+ if (!Cluster.ForwardThunkList.empty())
+ Insertions.push_back({Cluster.LastFunctionIndex + 1,
+ /*IsForward=*/true, &Cluster.ForwardThunkList});
+ }
+
+ // Apply insertions from high to low indices so earlier insertions do not
+ // invalidate later positions. At a shared boundary, repeated insertion at the
+ // same index reverses application order, yielding forward thunks before
+ // backward thunks.
+ llvm::sort(Insertions, [](const ThunkInsertion &A, const ThunkInsertion &B) {
+ if (A.Position != B.Position)
+ return A.Position > B.Position;
+ return !A.IsForward && B.IsForward;
+ });
+
+ for (ThunkInsertion &Insertion : Insertions) {
OutputFunctions.insert(
- std::next(OutputFunctions.begin(), FC.LastFunctionIndex + 1),
- FC.ThunkList.begin(), FC.ThunkList.end());
+ std::next(OutputFunctions.begin(), Insertion.Position),
+ Insertion.ThunkList->begin(), Insertion.ThunkList->end());
}
+}
+
+void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
+ BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
+ FragmentClusterLayout Layout = buildClusterLayout(BC, OutputFunctions);
+
+ if (Layout.Clusters.empty())
+ return;
+
+ relaxCalls(BC, OutputFunctions, Layout);
+ relaxUnconditionalBranches(BC, OutputFunctions, Layout);
+ insertClusterThunks(OutputFunctions, Layout);
LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
for (const auto *BF : OutputFunctions) { dbgs() << *BF << '\n'; });
@@ -1375,7 +1512,7 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
return Error::success();
BC.outs() << "BOLT-INFO: starting experimental relaxation pass\n";
- relaxCalls(BC);
+ relaxWithClusters(BC);
return Error::success();
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 512af53f0ec59..1d1728b868bdf 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4363,111 +4363,17 @@ void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
}
}
-namespace {
-
-/// Defines the strict weak ordering for BOLT-produced code sections.
-class CodeSectionOrder {
-public:
- CodeSectionOrder(StringRef ColdSectionName, StringRef HotTextMoverSectionName,
- StringRef MainSectionName, StringRef WarmSectionName,
- bool HotText, bool HotFunctionsAtEnd)
- : ColdSectionName(ColdSectionName),
- HotTextMoverSectionName(HotTextMoverSectionName),
- MainSectionName(MainSectionName), WarmSectionName(WarmSectionName),
- HotText(HotText), HotFunctionsAtEnd(HotFunctionsAtEnd) {}
-
- bool operator()(StringRef AName, StringRef BName) const {
- const SectionKind AKind = getKind(AName);
- const SectionKind BKind = getKind(BName);
- const unsigned ARank = getRank(AKind);
- const unsigned BRank = getRank(BKind);
- if (ARank != BRank)
- return ARank < BRank;
-
- if (AKind == SectionKind::Cold) {
- if (AName.size() != BName.size())
- return HotFunctionsAtEnd ? AName.size() > BName.size()
- : AName.size() < BName.size();
- if (AName != BName)
- return HotFunctionsAtEnd ? AName > BName : AName < BName;
- }
-
- return false;
- }
-
-private:
- enum class SectionKind { Mover, Main, Warm, Cold, Other };
-
- SectionKind getKind(StringRef Name) const {
- if (HotText && Name == HotTextMoverSectionName)
- return SectionKind::Mover;
- if (Name == MainSectionName)
- return SectionKind::Main;
- if (Name == WarmSectionName)
- return SectionKind::Warm;
- if (Name.starts_with(ColdSectionName))
- return SectionKind::Cold;
- return SectionKind::Other;
- }
-
- unsigned getRank(SectionKind Kind) const {
- if (Kind == SectionKind::Mover)
- return 0;
- if (HotFunctionsAtEnd) {
- switch (Kind) {
- case SectionKind::Other:
- return 1;
- case SectionKind::Cold:
- return 2;
- case SectionKind::Warm:
- return 3;
- case SectionKind::Main:
- return 4;
- case SectionKind::Mover:
- llvm_unreachable("handled above");
- }
- }
- switch (Kind) {
- case SectionKind::Main:
- return 1;
- case SectionKind::Warm:
- return 2;
- case SectionKind::Cold:
- return 3;
- case SectionKind::Other:
- return 4;
- case SectionKind::Mover:
- llvm_unreachable("handled above");
- }
- llvm_unreachable("unknown section kind");
- }
-
- StringRef ColdSectionName;
- StringRef HotTextMoverSectionName;
- StringRef MainSectionName;
- StringRef WarmSectionName;
- bool HotText;
- bool HotFunctionsAtEnd;
-};
-
-} // namespace
-
std::vector<BinarySection *> RewriteInstance::getCodeSections() {
std::vector<BinarySection *> CodeSections;
for (BinarySection &Section : BC->textSections())
if (Section.hasValidSectionID())
CodeSections.emplace_back(&Section);
- const CodeSectionOrder CompareSections(
- BC->getColdCodeSectionName(), BC->getHotTextMoverSectionName(),
- BC->getMainCodeSectionName(), BC->getWarmCodeSectionName(), opts::HotText,
- opts::HotFunctionsAtEnd);
-
// Determine the order of sections.
- llvm::stable_sort(CodeSections,
- [&](const BinarySection *A, const BinarySection *B) {
- return CompareSections(A->getName(), B->getName());
- });
+ llvm::stable_sort(
+ CodeSections, [&](const BinarySection *A, const BinarySection *B) {
+ return BC->compareSectionNames(A->getName(), B->getName());
+ });
#ifndef NDEBUG
// Verify that the order of sections and functions is consistent.
@@ -4477,7 +4383,7 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
uint32_t LastIndex = 0;
for (const BinaryFunction *BF : BC->getOutputBinaryFunctions()) {
- if (!BF->isEmitted() || BF->isPatch())
+ if (!BF->isEmitted() || BF->isPatch() || BF->isThunk())
continue;
ErrorOr<BinarySection &> Sec = BF->getCodeSection();
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
new file mode 100644
index 0000000000000..6c46afdb1c40a
--- /dev/null
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -0,0 +1,309 @@
+## Check branch thunk chains with adjacent fragment clusters. Split functions
+## have small duplicated constant islands, while hot-only pad functions create
+## most of the hot-section distance.
+##
+## Input layout:
+##
+## A 8MiB island, pad_hot_0 50MiB, B 8MiB island, pad_hot_1 50MiB,
+## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
+##
+## With --split-functions, BOLT places the cold blocks of A/B/C/D in
+## .text.cold. With the default 120MiB function-fragment cluster size, branch
+## relaxation sees:
+##
+## normal layout:
+## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
+## cluster 1, .text: C, pad_hot_2, D, pad_hot_3
+## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
+##
+## --hot-functions-at-end:
+## cluster 0, .text.cold: A.cold, B.cold, C.cold, D.cold
+## cluster 1, .text: A, pad_hot_0, B, pad_hot_1
+## cluster 2, .text: C, pad_hot_2, D, pad_hot_3
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-strip --strip-unneeded %t
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions \
+# RUN: --compact-code-model --relax-exp \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-bolt %t -o %t.hfe.bolt --data %t.fdata --split-functions \
+# RUN: --compact-code-model --relax-exp --hot-functions-at-end \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
+# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_4,__AArch64BranchForwardThunk_5,__AArch64BranchForwardThunk_8,__AArch64BranchForwardThunk_10,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_3,__AArch64BranchBackwardThunk_6,__AArch64BranchBackwardThunk_7,__AArch64BranchBackwardThunk_9,__AArch64BranchBackwardThunk_11 \
+# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchForwardThunk_10,__AArch64BranchForwardThunk_11,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5,__AArch64BranchBackwardThunk_8,__AArch64BranchBackwardThunk_9 \
+# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
+
+# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
+# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
+
+# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
+
+# CHECK-SECTIONS: .text
+# CHECK-SECTIONS: .text.cold
+
+ .text
+ .globl A
+ .type A, %function
+A:
+.A_entry:
+# FDATA: 1 A #.A_entry# 100
+ cbz x0, .A_ret
+ b .A_cold
+.A_ret:
+# FDATA: 1 A #.A_ret# 100
+ ret
+.A_cold:
+ mov x0, #1
+ b .A_ret
+ .space 0x800000
+ .size A, .-A
+
+ .globl pad_hot_0
+ .type pad_hot_0, %function
+pad_hot_0:
+.pad_hot_0_entry:
+# FDATA: 1 pad_hot_0 #.pad_hot_0_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_0, .-pad_hot_0
+
+ .globl B
+ .type B, %function
+B:
+.B_entry:
+# FDATA: 1 B #.B_entry# 100
+ cbz x0, .B_ret
+ b .B_cold
+.B_ret:
+# FDATA: 1 B #.B_ret# 100
+ ret
+.B_cold:
+ mov x0, #2
+ b .B_ret
+ .space 0x800000
+ .size B, .-B
+
+ .globl pad_hot_1
+ .type pad_hot_1, %function
+pad_hot_1:
+.pad_hot_1_entry:
+# FDATA: 1 pad_hot_1 #.pad_hot_1_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_1, .-pad_hot_1
+
+ .globl C
+ .type C, %function
+C:
+.C_entry:
+# FDATA: 1 C #.C_entry# 100
+ cbz x0, .C_ret
+ b .C_cold
+.C_ret:
+# FDATA: 1 C #.C_ret# 100
+ ret
+.C_cold:
+ mov x0, #3
+ b .C_ret
+ .space 0x800000
+ .size C, .-C
+
+ .globl pad_hot_2
+ .type pad_hot_2, %function
+pad_hot_2:
+.pad_hot_2_entry:
+# FDATA: 1 pad_hot_2 #.pad_hot_2_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_2, .-pad_hot_2
+
+ .globl D
+ .type D, %function
+D:
+.D_entry:
+# FDATA: 1 D #.D_entry# 100
+ cbz x0, .D_ret
+ b .D_cold
+.D_ret:
+# FDATA: 1 D #.D_ret# 100
+ ret
+.D_cold:
+ mov x0, #4
+ b .D_ret
+ .space 0x800000
+ .size D, .-D
+
+ .globl pad_hot_3
+ .type pad_hot_3, %function
+pad_hot_3:
+.pad_hot_3_entry:
+# FDATA: 1 pad_hot_3 #.pad_hot_3_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_3, .-pad_hot_3
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# CHECK-OUTPUT: Disassembly of section .text:
+
+# CHECK-OUTPUT: <A>:
+# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[A_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: [[A_RET:[0-9a-f]+]]: {{.*}} ret
+# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+
+# CHECK-OUTPUT: <B>:
+# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[B_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: [[B_RET:[0-9a-f]+]]: {{.*}} ret
+# CHECK-OUTPUT-NEXT: [[B_BR]]: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_5>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_1>:
+# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_0>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_5>:
+# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_4>
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_2>:
+# CHECK-OUTPUT-NEXT: [[A_BW1:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_6>:
+# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x4>
+
+# CHECK-OUTPUT: <C>:
+# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[C_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: [[C_RET:[0-9a-f]+]]: {{.*}} ret
+# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_8>
+
+# CHECK-OUTPUT: <D>:
+# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
+# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_0>:
+# CHECK-OUTPUT-NEXT: [[A_FW1]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_4>:
+# CHECK-OUTPUT-NEXT: [[B_FW1]]: {{.*}} b 0x[[B_COLD:[0-9a-f]+]] <B.cold.0>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_8>:
+# CHECK-OUTPUT-NEXT: [[C_FW0]]: {{.*}} b 0x[[C_COLD:[0-9a-f]+]] <C.cold.0>
+
+# CHECK-OUTPUT: <__AArch64BranchForwardThunk_10>:
+# CHECK-OUTPUT-NEXT: [[D_FW0]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
+
+# CHECK-OUTPUT: Disassembly of section .text.cold:
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_3>:
+# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_BW1]] <__AArch64BranchBackwardThunk_2>
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_7>:
+# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64BranchBackwardThunk_6>
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_9>:
+# CHECK-OUTPUT-NEXT: [[C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[C_RET]] <C+0x4>
+
+# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_11>:
+# CHECK-OUTPUT-NEXT: [[D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[D_RET]] <D+0x4>
+
+# CHECK-OUTPUT: <A.cold.0>:
+# CHECK-OUTPUT-NEXT: [[A_COLD]]: {{.*}} mov x0, #0x1
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64BranchBackwardThunk_3>
+
+# CHECK-OUTPUT: <B.cold.0>:
+# CHECK-OUTPUT-NEXT: [[B_COLD]]: {{.*}} mov x0, #0x2
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64BranchBackwardThunk_7>
+
+# CHECK-OUTPUT: <C.cold.0>:
+# CHECK-OUTPUT-NEXT: [[C_COLD]]: {{.*}} mov x0, #0x3
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64BranchBackwardThunk_9>
+
+# CHECK-OUTPUT: <D.cold.0>:
+# CHECK-OUTPUT-NEXT: [[D_COLD]]: {{.*}} mov x0, #0x4
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[D_BW0]] <__AArch64BranchBackwardThunk_11>
+
+
+# CHECK-HFE-OUTPUT: Disassembly of section .text.cold:
+
+# CHECK-HFE-OUTPUT: <A.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x1
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+
+# CHECK-HFE-OUTPUT: <B.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
+
+# CHECK-HFE-OUTPUT: <C.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
+
+# CHECK-HFE-OUTPUT: <D.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_11>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_FW]]: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_7>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_11>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+
+# CHECK-HFE-OUTPUT: Disassembly of section .text:
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_4>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_8>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
+
+# CHECK-HFE-OUTPUT: <A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x{{[0-9a-f]+}} <__AArch64BranchBackwardThunk_0>
+
+# CHECK-HFE-OUTPUT: <B>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_B_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BR]]: {{.*}} b 0x{{[0-9a-f]+}} <__AArch64BranchBackwardThunk_2>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_6>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_10>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64BranchBackwardThunk_4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_9>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_8>
+
+# CHECK-HFE-OUTPUT: <C>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64BranchBackwardThunk_5>
+
+# CHECK-HFE-OUTPUT: <D>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_9>
diff --git a/bolt/test/AArch64/relax-exp.s b/bolt/test/AArch64/relax-calls.s
similarity index 86%
rename from bolt/test/AArch64/relax-exp.s
rename to bolt/test/AArch64/relax-calls.s
index f6394b7a39df7..3f2b0375e304b 100644
--- a/bolt/test/AArch64/relax-exp.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -55,17 +55,17 @@ hot:
# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
## Check the number of thunks created in other modes.
-# CHECK-BOLT: BOLT-INFO: 4 short thunks created
-# CHECK-BOLT: BOLT-INFO: 3 long thunks created
+# CHECK-BOLT: BOLT-INFO: 5 short thunks created
+# CHECK-BOLT: BOLT-INFO: 5 long thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 4 short thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 6 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 4 long thunks created
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_foo>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk_bar>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_bar>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk__start>
.global _start
.type _start, %function
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
new file mode 100644
index 0000000000000..f99d30ee94f8d
--- /dev/null
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -0,0 +1,216 @@
+## Check call relaxation with function fragment clusters. This uses split
+## functions with calls from hot and cold fragments so call thunks are placed
+## around the clusters.
+##
+## Input layout:
+##
+## A 8MiB island, pad_hot_0 50MiB, B 8MiB island, pad_hot_1 50MiB,
+## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
+##
+## With --split-functions, BOLT places the cold blocks of A/B/C/D in
+## .text.cold. With the default 120MiB function-fragment cluster size, call
+## relaxation sees:
+##
+## normal layout:
+## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
+## cluster 1, .text: C, pad_hot_2, D, pad_hot_3
+## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
+##
+## --hot-functions-at-end:
+## cluster 0, .text.cold: A.cold, B.cold, C.cold, D.cold
+## cluster 1, .text: A, pad_hot_0, B, pad_hot_1
+## cluster 2, .text: C, pad_hot_2, D, pad_hot_3
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-strip --strip-unneeded %t
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions \
+# RUN: --compact-code-model --relax-exp \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-bolt %t -o %t.hfe.bolt --data %t.fdata --split-functions \
+# RUN: --compact-code-model --relax-exp --hot-functions-at-end \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
+# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64ADRPThunk_A \
+# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64ADRPThunk_C \
+# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
+
+# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
+# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with thunks
+# CHECK-BOLT: BOLT-INFO: 3 short thunks created
+# CHECK-BOLT: BOLT-INFO: 1 long thunks created
+# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
+
+# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
+
+# CHECK-SECTIONS: .text
+# CHECK-SECTIONS: .text.cold
+
+ .text
+ .globl A
+ .type A, %function
+A:
+.A_entry:
+# FDATA: 1 A #.A_entry# 100
+ bl B
+ bl C
+ cbz x0, .A_ret
+ b .A_cold
+.A_ret:
+# FDATA: 1 A #.A_ret# 100
+ ret
+.A_cold:
+ mov x0, #1
+ bl C
+ bl A
+ b .A_ret
+ .space 0x800000
+ .size A, .-A
+
+ .globl pad_hot_0
+ .type pad_hot_0, %function
+pad_hot_0:
+.pad_hot_0_entry:
+# FDATA: 1 pad_hot_0 #.pad_hot_0_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_0, .-pad_hot_0
+
+ .globl B
+ .type B, %function
+B:
+.B_entry:
+# FDATA: 1 B #.B_entry# 100
+ cbz x0, .B_ret
+ b .B_cold
+.B_ret:
+# FDATA: 1 B #.B_ret# 100
+ ret
+.B_cold:
+ mov x0, #2
+ b .B_ret
+ .space 0x800000
+ .size B, .-B
+
+ .globl pad_hot_1
+ .type pad_hot_1, %function
+pad_hot_1:
+.pad_hot_1_entry:
+# FDATA: 1 pad_hot_1 #.pad_hot_1_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_1, .-pad_hot_1
+
+ .globl C
+ .type C, %function
+C:
+.C_entry:
+# FDATA: 1 C #.C_entry# 100
+ bl A
+ cbz x0, .C_ret
+ b .C_cold
+.C_ret:
+# FDATA: 1 C #.C_ret# 100
+ ret
+.C_cold:
+ mov x0, #3
+ b .C_ret
+ .space 0x800000
+ .size C, .-C
+
+ .globl pad_hot_2
+ .type pad_hot_2, %function
+pad_hot_2:
+.pad_hot_2_entry:
+# FDATA: 1 pad_hot_2 #.pad_hot_2_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_2, .-pad_hot_2
+
+ .globl D
+ .type D, %function
+D:
+.D_entry:
+# FDATA: 1 D #.D_entry# 100
+ cbz x0, .D_ret
+ b .D_cold
+.D_ret:
+# FDATA: 1 D #.D_ret# 100
+ ret
+.D_cold:
+ mov x0, #4
+ b .D_ret
+ .space 0x800000
+ .size D, .-D
+
+ .globl pad_hot_3
+ .type pad_hot_3, %function
+pad_hot_3:
+.pad_hot_3_entry:
+# FDATA: 1 pad_hot_3 #.pad_hot_3_entry# 100
+ ret
+ .space 0x3200000
+ .size pad_hot_3, .-pad_hot_3
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# CHECK-OUTPUT: <A>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+
+# CHECK-OUTPUT: <__AArch64Thunk_C>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
+
+# CHECK-OUTPUT: <__AArch64Thunk_A>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+
+# CHECK-OUTPUT: <C>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+
+# CHECK-OUTPUT: <__AArch64ADRPThunk_A>:
+# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}} <A>
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, #0x0
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-OUTPUT: <A.cold.0>:
+# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x1
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_A>
+
+# CHECK-HFE-OUTPUT: <A.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x1
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_C>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+
+# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_C>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, #0x{{[0-9a-f]+}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-HFE-OUTPUT: <__AArch64Thunk_A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+
+# CHECK-HFE-OUTPUT: <A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+
+# CHECK-HFE-OUTPUT: <__AArch64Thunk_C>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
+
+# CHECK-HFE-OUTPUT: <__AArch64Thunk_A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+
+# CHECK-HFE-OUTPUT: <C>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
>From 6514e5993db0a696630c50fdc53432f088e3d356 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 27 Aug 2026 09:00:27 +0000
Subject: [PATCH 02/22] allow clusters to span across sections
---
bolt/include/bolt/Passes/LongJmp.h | 18 +++--
bolt/lib/Passes/LongJmp.cpp | 15 ++--
.../AArch64/relax-branches-with-thunk-chain.s | 73 +++++++------------
bolt/test/AArch64/relax-calls.s | 14 ++--
.../test/AArch64/relax-cross-fragment-calls.s | 34 ++++-----
5 files changed, 65 insertions(+), 89 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 37c7e1452d5bf..e021e8fad503b 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -84,10 +84,13 @@ class LongJmpPass : public BinaryFunctionPass {
/// A group of function fragments that are located within the longest direct
/// branch/call instruction distance. Jumps within the cluster do not require
- /// a thunk. The cluster may include thunks for jumps to targets outside.
+ /// a thunk. The cluster may span output sections and include thunks for jumps
+ /// to targets outside. Backward thunks are inserted before the cluster, while
+ /// forward thunks are inserted after it.
struct FragmentCluster {
- /// Output code section containing fragments in this cluster.
- SmallString<32> SectionName;
+ /// Output code sections containing the first and last cluster fragments.
+ SmallString<32> StartSectionName;
+ SmallString<32> EndSectionName;
/// Estimated size of the cluster in bytes.
uint64_t Size{0};
@@ -115,12 +118,11 @@ class LongJmpPass : public BinaryFunctionPass {
/// <Destination Symbol> -> <Thunk Function>.
DenseMap<const MCSymbol *, BinaryFunction *> ForwardBranchThunks;
DenseMap<const MCSymbol *, BinaryFunction *> BackwardBranchThunks;
- };
- /// Maximum size of combined regular functions in the cluster. Note that it's
- /// less than 128MB, because the size of the cluster plus its thunks should be
- /// less than 128MB.
- static constexpr uint64_t MaxClusterSize = 120 * 1024 * 1024;
+ StringRef getThunkSectionName(bool IsForward) const {
+ return IsForward ? EndSectionName : StartSectionName;
+ }
+ };
struct FragmentClusterLayout {
SmallVector<FragmentCluster, 4> Clusters;
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index ccdb2ef37f77f..3a0dda76e0b08 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -40,6 +40,11 @@ static cl::opt<bool>
static cl::opt<bool> RelaxPLT("relax-plt",
cl::desc("indicate PLT proximity to hot text"),
cl::init(true), cl::cat(BoltOptCategory));
+
+static cl::opt<unsigned long long> MaxClusterSize(
+ "max-cluster-size",
+ cl::desc("maximum estimated size of a function fragment cluster in bytes"),
+ cl::init(124 * 1024 * 1024), cl::cat(BoltOptCategory));
}
namespace llvm {
@@ -1079,15 +1084,15 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
const uint64_t FFSize = estimateFragmentSize(BF, FF);
if (Layout.Clusters.empty() ||
- Layout.Clusters.back().SectionName != Fragment.SectionName ||
- Layout.Clusters.back().Size + FFSize > MaxClusterSize) {
+ Layout.Clusters.back().Size + FFSize > opts::MaxClusterSize) {
Layout.Clusters.emplace_back(FragmentCluster());
FragmentCluster &FC = Layout.Clusters.back();
- FC.SectionName = Fragment.SectionName;
+ FC.StartSectionName = Fragment.SectionName;
FC.FirstFunctionIndex = Fragment.FunctionIndex;
}
FragmentCluster &FC = Layout.Clusters.back();
+ FC.EndSectionName = Fragment.SectionName;
FC.LastFunctionIndex = Fragment.FunctionIndex;
++FC.NumFragments;
EstimatedSize += FFSize;
@@ -1249,7 +1254,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort);
FragmentCluster &ThunkCluster = Clusters[SourceCluster];
- Thunk->setCodeSectionName(ThunkCluster.SectionName);
+ Thunk->setCodeSectionName(ThunkCluster.getThunkSectionName(IsForward));
BinaryFunctionListType &ThunkList = IsForward
? ThunkCluster.ForwardThunkList
: ThunkCluster.BackwardThunkList;
@@ -1355,7 +1360,7 @@ void LongJmpPass::relaxUnconditionalBranches(
return It->second;
BinaryFunction *Thunk = createBranchThunk(NextTarget, IsForward);
- Thunk->setCodeSectionName(Cluster.SectionName);
+ Thunk->setCodeSectionName(Cluster.getThunkSectionName(IsForward));
auto &ThunkList =
IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
ThunkList.push_back(Thunk);
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index 6c46afdb1c40a..1eddf8160fca9 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -8,8 +8,8 @@
## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
##
## With --split-functions, BOLT places the cold blocks of A/B/C/D in
-## .text.cold. With the default 120MiB function-fragment cluster size, branch
-## relaxation sees:
+## .text.cold. With the default 124MiB function-fragment cluster size,
+## branch relaxation sees:
##
## normal layout:
## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
@@ -17,9 +17,10 @@
## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
##
## --hot-functions-at-end:
-## cluster 0, .text.cold: A.cold, B.cold, C.cold, D.cold
-## cluster 1, .text: A, pad_hot_0, B, pad_hot_1
-## cluster 2, .text: C, pad_hot_2, D, pad_hot_3
+## cluster 0: .text.cold A.cold, B.cold, C.cold, D.cold
+## .text A, pad_hot_0, B
+## cluster 1: .text pad_hot_1, C, pad_hot_2, D
+## cluster 2: .text pad_hot_3
# REQUIRES: system-linux
@@ -37,7 +38,7 @@
# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_4,__AArch64BranchForwardThunk_5,__AArch64BranchForwardThunk_8,__AArch64BranchForwardThunk_10,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_3,__AArch64BranchBackwardThunk_6,__AArch64BranchBackwardThunk_7,__AArch64BranchBackwardThunk_9,__AArch64BranchBackwardThunk_11 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchForwardThunk_10,__AArch64BranchForwardThunk_11,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5,__AArch64BranchBackwardThunk_8,__AArch64BranchBackwardThunk_9 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -45,8 +46,8 @@
# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 4 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -236,74 +237,50 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x4>
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_11>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_FW]]: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x4>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_7>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_11>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_4>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_8>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
-
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x{{[0-9a-f]+}} <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_B_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BR]]: {{.*}} b 0x{{[0-9a-f]+}} <__AArch64BranchBackwardThunk_2>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BR]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_6>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_10>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64BranchBackwardThunk_4>
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_9>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_8>
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64BranchBackwardThunk_5>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW]] <__AArch64BranchBackwardThunk_0>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_9>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW]] <__AArch64BranchBackwardThunk_2>
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 3f2b0375e304b..767195f648c4a 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -55,17 +55,17 @@ hot:
# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
## Check the number of thunks created in other modes.
-# CHECK-BOLT: BOLT-INFO: 5 short thunks created
-# CHECK-BOLT: BOLT-INFO: 5 long thunks created
+# CHECK-BOLT: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT: BOLT-INFO: 3 long thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 6 short thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 4 long thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 2 long thunks created
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_foo>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_bar>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk__start>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <foo>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk_bar>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk__start>
.global _start
.type _start, %function
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index f99d30ee94f8d..b1522e2db38bc 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -8,8 +8,8 @@
## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
##
## With --split-functions, BOLT places the cold blocks of A/B/C/D in
-## .text.cold. With the default 120MiB function-fragment cluster size, call
-## relaxation sees:
+## .text.cold. With the default 124MiB function-fragment cluster size,
+## call relaxation sees:
##
## normal layout:
## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
@@ -17,9 +17,10 @@
## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
##
## --hot-functions-at-end:
-## cluster 0, .text.cold: A.cold, B.cold, C.cold, D.cold
-## cluster 1, .text: A, pad_hot_0, B, pad_hot_1
-## cluster 2, .text: C, pad_hot_2, D, pad_hot_3
+## cluster 0: .text.cold A.cold, B.cold, C.cold, D.cold
+## .text A, pad_hot_0, B
+## cluster 1: .text pad_hot_1, C, pad_hot_2, D
+## cluster 2: .text pad_hot_3
# REQUIRES: system-linux
@@ -37,7 +38,7 @@
# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64ADRPThunk_A \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64ADRPThunk_C \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -48,11 +49,10 @@
# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 calls with thunks
-# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: 2 short thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 4 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -191,16 +191,8 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_C>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
-
-# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_C>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, #0x{{[0-9a-f]+}}
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
-
-# CHECK-HFE-OUTPUT: <__AArch64Thunk_A>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
>From b88d89dcf62a093deca20e88f17ddf59b5443559 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 27 Aug 2026 12:46:05 +0000
Subject: [PATCH 03/22] unify chain creation loops
---
bolt/lib/Passes/LongJmp.cpp | 25 +++++++++----------------
bolt/test/AArch64/relax-calls.s | 4 ++--
2 files changed, 11 insertions(+), 18 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 3a0dda76e0b08..239dfc4cfe007 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1375,22 +1375,15 @@ void LongJmpPass::relaxUnconditionalBranches(
BinaryFunction *FirstThunk = nullptr;
const MCSymbol *NextTarget = Branch.TargetSymbol;
- if (SourceCluster < TargetCluster) {
- for (unsigned Cluster = TargetCluster; Cluster > SourceCluster;) {
- --Cluster;
- FirstThunk =
- getOrCreateBranchThunk(Clusters[Cluster], Branch.TargetSymbol,
- NextTarget, /*IsForward=*/true);
- NextTarget = FirstThunk->getSymbol();
- }
- } else {
- for (unsigned Cluster = TargetCluster + 1; Cluster <= SourceCluster;
- ++Cluster) {
- FirstThunk =
- getOrCreateBranchThunk(Clusters[Cluster], Branch.TargetSymbol,
- NextTarget, /*IsForward=*/false);
- NextTarget = FirstThunk->getSymbol();
- }
+ const bool IsForward = SourceCluster < TargetCluster;
+ const unsigned NumHops = IsForward ? TargetCluster - SourceCluster
+ : SourceCluster - TargetCluster;
+ for (unsigned I = 0; I < NumHops; ++I) {
+ const unsigned Cluster =
+ IsForward ? TargetCluster - I - 1 : TargetCluster + I + 1;
+ FirstThunk = getOrCreateBranchThunk(
+ Clusters[Cluster], Branch.TargetSymbol, NextTarget, IsForward);
+ NextTarget = FirstThunk->getSymbol();
}
assert(FirstThunk && "expected branch thunk chain");
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 767195f648c4a..f6394b7a39df7 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -63,9 +63,9 @@ hot:
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
-# CHECK-OUTPUT-NEXT: bl {{.*}} <foo>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_foo>
# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk_bar>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk__start>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
.global _start
.type _start, %function
>From ab3f49f2e2d221a7ee73911b41110f9ce43aa962 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 1 Sep 2026 12:44:11 +0000
Subject: [PATCH 04/22] Reuse adjacent long call thunks
---
bolt/lib/Passes/LongJmp.cpp | 99 +++++++--
.../AArch64/relax-branches-with-thunk-chain.s | 113 +++++-----
.../test/AArch64/relax-cross-fragment-calls.s | 197 +++++++++++-------
3 files changed, 264 insertions(+), 145 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 239dfc4cfe007..71ff27ccc4d0e 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1169,7 +1169,14 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
unsigned TargetCluster;
};
- SmallVector<CrossClusterCall> CrossClusterCalls;
+ auto clusterDistance = [](const CrossClusterCall &Call) {
+ return Call.SourceCluster > Call.TargetCluster
+ ? Call.SourceCluster - Call.TargetCluster
+ : Call.TargetCluster - Call.SourceCluster;
+ };
+
+ SmallVector<CrossClusterCall> AdjacentClusterCalls;
+ SmallVector<CrossClusterCall> RemoteClusterCalls;
for (BinaryFunction *BF : OutputFunctions) {
if (!BC.shouldEmit(*BF) || BF->isPatch())
continue;
@@ -1204,14 +1211,19 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
if (TargetCluster == SourceCluster)
continue;
- CrossClusterCalls.push_back(
- {&Inst, TargetSymbol, SourceCluster, TargetCluster});
+ CrossClusterCall Call{&Inst, TargetSymbol, SourceCluster,
+ TargetCluster};
+ auto &Calls = clusterDistance(Call) == 1 ? AdjacentClusterCalls
+ : RemoteClusterCalls;
+ Calls.push_back(Call);
}
}
}
size_t NumShortThunks = 0;
size_t NumLongThunks = 0;
+ size_t NumShortThunksReused = 0;
+ size_t NumLongThunksReused = 0;
auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort) {
BinaryFunction *Thunk = nullptr;
if (IsShort) {
@@ -1248,8 +1260,37 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
unsigned SourceCluster, bool IsShort,
bool IsForward) {
FragmentCluster &FC = Clusters[SourceCluster];
- if (auto It = FC.CallThunks.find(TargetSymbol); It != FC.CallThunks.end())
+ if (auto It = FC.CallThunks.find(TargetSymbol); It != FC.CallThunks.end()) {
+ if (IsShort)
+ ++NumShortThunksReused;
+ else
+ ++NumLongThunksReused;
return It->second;
+ }
+
+ // Reuse long thunks hosted at the adjacent cluster boundary.
+ if (!IsShort) {
+ FragmentCluster *ReuseCluster = nullptr;
+ BinaryFunctionListType *ReuseThunkList = nullptr;
+ if (IsForward && SourceCluster > 0) {
+ ReuseCluster = &Clusters[SourceCluster - 1];
+ ReuseThunkList = &ReuseCluster->ForwardThunkList;
+ } else if (!IsForward && SourceCluster + 1 < Clusters.size()) {
+ ReuseCluster = &Clusters[SourceCluster + 1];
+ ReuseThunkList = &ReuseCluster->BackwardThunkList;
+ }
+
+ if (ReuseCluster) {
+ auto It = ReuseCluster->CallThunks.find(TargetSymbol);
+ if (It != ReuseCluster->CallThunks.end() &&
+ llvm::is_contained(*ReuseThunkList, It->second)) {
+ BinaryFunction *Thunk = It->second;
+ ++NumLongThunksReused;
+ registerCallThunk(FC, TargetSymbol, Thunk);
+ return Thunk;
+ }
+ }
+ }
BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort);
@@ -1265,28 +1306,47 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
return Thunk;
};
- auto areAdjacent = [](unsigned A, unsigned B) {
- return A > B ? A - B == 1 : B - A == 1;
- };
+ // Process farthest calls first so closer remote calls can reuse long thunks
+ // already placed at adjacent cluster boundaries.
+ llvm::stable_sort(RemoteClusterCalls,
+ [&](const CrossClusterCall &A, const CrossClusterCall &B) {
+ return clusterDistance(A) > clusterDistance(B);
+ });
- for (CrossClusterCall &Call : CrossClusterCalls) {
+ auto relaxCall = [&](CrossClusterCall &Call, bool IsShort) {
const bool IsForward = Call.SourceCluster < Call.TargetCluster;
- const bool Adjacent = areAdjacent(Call.SourceCluster, Call.TargetCluster);
-
BinaryFunction *Thunk = getOrCreateCallThunk(
- Call.TargetSymbol, Call.SourceCluster, Adjacent, IsForward);
+ Call.TargetSymbol, Call.SourceCluster, IsShort, IsForward);
BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
- }
+ };
+
+ for (CrossClusterCall &Call : AdjacentClusterCalls)
+ relaxCall(Call, /*IsShort=*/true);
+
+ for (CrossClusterCall &Call : RemoteClusterCalls)
+ relaxCall(Call, /*IsShort=*/false);
- if (!CrossClusterCalls.empty())
- BC.outs() << "BOLT-INFO: relaxed " << CrossClusterCalls.size()
- << " calls with thunks\n";
+ if (!AdjacentClusterCalls.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << AdjacentClusterCalls.size()
+ << " adjacent cluster calls with thunks\n";
+
+ if (!RemoteClusterCalls.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << RemoteClusterCalls.size()
+ << " remote cluster calls with thunks\n";
if (NumShortThunks)
BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
if (NumLongThunks)
BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
+
+ if (NumShortThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumShortThunksReused
+ << " short thunks reused\n";
+
+ if (NumLongThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumLongThunksReused
+ << " long thunks reused\n";
}
void LongJmpPass::relaxUnconditionalBranches(
@@ -1337,6 +1397,7 @@ void LongJmpPass::relaxUnconditionalBranches(
}
size_t NumBranchThunks = 0;
+ size_t NumBranchThunksReused = 0;
auto createBranchThunk = [&](const MCSymbol *TargetSymbol,
const bool IsForward) {
std::string ThunkName = IsForward ? "__AArch64BranchForwardThunk_"
@@ -1356,8 +1417,10 @@ void LongJmpPass::relaxUnconditionalBranches(
auto &Thunks = IsForward ? Cluster.ForwardBranchThunks
: Cluster.BackwardBranchThunks;
auto It = Thunks.find(TargetSymbol);
- if (It != Thunks.end())
+ if (It != Thunks.end()) {
+ ++NumBranchThunksReused;
return It->second;
+ }
BinaryFunction *Thunk = createBranchThunk(NextTarget, IsForward);
Thunk->setCodeSectionName(Cluster.getThunkSectionName(IsForward));
@@ -1401,6 +1464,10 @@ void LongJmpPass::relaxUnconditionalBranches(
if (NumBranchThunks)
BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
+
+ if (NumBranchThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunksReused
+ << " branch thunks reused\n";
}
void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index 1eddf8160fca9..5b2176d70fe33 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -1,14 +1,9 @@
-## Check branch thunk chains with adjacent fragment clusters. Split functions
-## have small duplicated constant islands, while hot-only pad functions create
-## most of the hot-section distance.
-##
-## Input layout:
-##
-## A 8MiB island, pad_hot_0 50MiB, B 8MiB island, pad_hot_1 50MiB,
-## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
-##
-## With --split-functions, BOLT places the cold blocks of A/B/C/D in
-## .text.cold. With the default 124MiB function-fragment cluster size,
+## Relax unconditional branches using thunk chains across fragment clusters.
+## A/B/C/D contain 16MiB islands and are interleaved with 40MiB hot pad functions.
+## This deliberately tests that hot and cold fragments may coexist in a cluster
+## when --hot-functions-at-end is specified. B has duplicate hot-to-cold branches
+## which share a forward chain in the normal layout and a backward chain with
+## --hot-functions-at-end. With the default 124MiB function-fragment cluster size,
## branch relaxation sees:
##
## normal layout:
@@ -18,9 +13,9 @@
##
## --hot-functions-at-end:
## cluster 0: .text.cold A.cold, B.cold, C.cold, D.cold
-## .text A, pad_hot_0, B
-## cluster 1: .text pad_hot_1, C, pad_hot_2, D
-## cluster 2: .text pad_hot_3
+## .text A, pad_hot_0
+## cluster 1: .text B, pad_hot_1, C, pad_hot_2
+## cluster 2: .text D, pad_hot_3
# REQUIRES: system-linux
@@ -38,16 +33,18 @@
# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_4,__AArch64BranchForwardThunk_5,__AArch64BranchForwardThunk_8,__AArch64BranchForwardThunk_10,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_3,__AArch64BranchBackwardThunk_6,__AArch64BranchBackwardThunk_7,__AArch64BranchBackwardThunk_9,__AArch64BranchBackwardThunk_11 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: relaxed 9 cross-cluster branches
# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
+# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 4 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 7 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 8 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 1 branch thunks reused
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -66,7 +63,7 @@ A:
.A_cold:
mov x0, #1
b .A_ret
- .space 0x800000
+ .space 0x1000000
.size A, .-A
.globl pad_hot_0
@@ -75,7 +72,7 @@ pad_hot_0:
.pad_hot_0_entry:
# FDATA: 1 pad_hot_0 #.pad_hot_0_entry# 100
ret
- .space 0x3200000
+ .space 0x2800000
.size pad_hot_0, .-pad_hot_0
.globl B
@@ -83,15 +80,19 @@ pad_hot_0:
B:
.B_entry:
# FDATA: 1 B #.B_entry# 100
+ tbz w0, #0, .B_alt
cbz x0, .B_ret
b .B_cold
+.B_alt:
+# FDATA: 1 B #.B_alt# 100
+ b .B_cold
.B_ret:
# FDATA: 1 B #.B_ret# 100
ret
.B_cold:
mov x0, #2
b .B_ret
- .space 0x800000
+ .space 0x1000000
.size B, .-B
.globl pad_hot_1
@@ -100,7 +101,7 @@ pad_hot_1:
.pad_hot_1_entry:
# FDATA: 1 pad_hot_1 #.pad_hot_1_entry# 100
ret
- .space 0x3200000
+ .space 0x2800000
.size pad_hot_1, .-pad_hot_1
.globl C
@@ -116,7 +117,7 @@ C:
.C_cold:
mov x0, #3
b .C_ret
- .space 0x800000
+ .space 0x1000000
.size C, .-C
.globl pad_hot_2
@@ -125,7 +126,7 @@ pad_hot_2:
.pad_hot_2_entry:
# FDATA: 1 pad_hot_2 #.pad_hot_2_entry# 100
ret
- .space 0x3200000
+ .space 0x2800000
.size pad_hot_2, .-pad_hot_2
.globl D
@@ -141,7 +142,7 @@ D:
.D_cold:
mov x0, #4
b .D_ret
- .space 0x800000
+ .space 0x1000000
.size D, .-D
.globl pad_hot_3
@@ -150,7 +151,7 @@ pad_hot_3:
.pad_hot_3_entry:
# FDATA: 1 pad_hot_3 #.pad_hot_3_entry# 100
ret
- .space 0x3200000
+ .space 0x2800000
.size pad_hot_3, .-pad_hot_3
## Force relocation mode.
@@ -161,12 +162,14 @@ pad_hot_3:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
# CHECK-OUTPUT: <B>:
-# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[B_BR:[0-9a-f]+]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: [[B_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[B_BR]]: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_5>
+# CHECK-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[B_ALT:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[B_RET:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_5>
+# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64BranchForwardThunk_5>
+# CHECK-OUTPUT-NEXT: [[B_RET]]: {{.*}} ret
# CHECK-OUTPUT: <__AArch64BranchForwardThunk_1>:
# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_0>
@@ -178,17 +181,17 @@ pad_hot_3:
# CHECK-OUTPUT-NEXT: [[A_BW1:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_6>:
-# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x4>
+# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x10>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[C_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_8>
+# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_8>
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
# CHECK-OUTPUT: <__AArch64BranchForwardThunk_0>:
# CHECK-OUTPUT-NEXT: [[A_FW1]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
@@ -241,46 +244,60 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x4>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
-
-# CHECK-HFE-OUTPUT: <B>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_B_BR:[0-9a-f]+]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BR]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x10>
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_7>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_4>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
+
+# CHECK-HFE-OUTPUT: <B>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[HFE_B_ALT:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[HFE_B_RET]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW]] <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW]] <__AArch64BranchBackwardThunk_2>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_6>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_4>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW]] <__AArch64BranchBackwardThunk_2>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_5>
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index b1522e2db38bc..6739cd1437049 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -1,26 +1,33 @@
-## Check call relaxation with function fragment clusters. This uses split
-## functions with calls from hot and cold fragments so call thunks are placed
-## around the clusters.
+## Relax direct calls using thunks across function fragment clusters. A/B/C/D
+## contain 56MiB islands. With --split-functions, BOLT places the cold blocks
+## of A/B/C/D in .text.cold. The test exercises forward and backward short
+## thunks for adjacent clusters, as well as long thunk reuse across remote
+## clusters. With the default 124MiB function-fragment cluster size, call
+## relaxation sees:
##
-## Input layout:
+## A -> B same cluster, direct
+## B -> C forward short thunk
+## C -> A backward short thunk
+## D -> A shares backward short thunk to A
##
-## A 8MiB island, pad_hot_0 50MiB, B 8MiB island, pad_hot_1 50MiB,
-## C 8MiB island, pad_hot_2 50MiB, D 8MiB island, pad_hot_3 50MiB
+## B.cold -> B normal: backward long thunk
+## C.cold -> B normal: shares backward long thunk to B
+## D.cold -> B normal: shares backward long thunk to B
##
-## With --split-functions, BOLT places the cold blocks of A/B/C/D in
-## .text.cold. With the default 124MiB function-fragment cluster size,
-## call relaxation sees:
+## B.cold -> D HFE: forward long thunk
+## D.cold -> D HFE: shares forward long thunk to D
##
## normal layout:
-## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
-## cluster 1, .text: C, pad_hot_2, D, pad_hot_3
-## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
+## cluster 0, .text: A, B
+## cluster 1, .text: C, D
+## cluster 2, .text.cold: A.cold, B.cold
+## cluster 3, .text.cold: C.cold, D.cold
##
## --hot-functions-at-end:
-## cluster 0: .text.cold A.cold, B.cold, C.cold, D.cold
-## .text A, pad_hot_0, B
-## cluster 1: .text pad_hot_1, C, pad_hot_2, D
-## cluster 2: .text pad_hot_3
+## cluster 0: .text.cold A.cold, B.cold
+## cluster 1: .text.cold C.cold, D.cold
+## cluster 2: .text A, B
+## cluster 3: .text C, D
# REQUIRES: system-linux
@@ -35,24 +42,31 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64ADRPThunk_A \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64Thunk_D,__AArch64ADRPThunk_B,__AArch64ADRPThunk_D \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_B,__AArch64Thunk_C,__AArch64ADRPThunk_B,__AArch64ADRPThunk_D \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
-# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with thunks
+# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
+# CHECK-BOLT: BOLT-INFO: relaxed 4 adjacent cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: 3 short thunks created
-# CHECK-BOLT: BOLT-INFO: 1 long thunks created
+# CHECK-BOLT: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT: BOLT-INFO: 1 short thunks reused
+# CHECK-BOLT: BOLT-INFO: 2 long thunks reused
# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
-# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
-
-# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 calls with thunks
-# CHECK-BOLT-HFE: BOLT-INFO: 2 short thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 4 branch thunks created
+# CHECK-BOLT: BOLT-INFO: 16 branch thunks created
+
+# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 5 adjacent cluster calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 remote cluster calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 2 short thunks reused
+# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks reused
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 16 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -64,7 +78,6 @@ A:
.A_entry:
# FDATA: 1 A #.A_entry# 100
bl B
- bl C
cbz x0, .A_ret
b .A_cold
.A_ret:
@@ -72,26 +85,16 @@ A:
ret
.A_cold:
mov x0, #1
- bl C
- bl A
b .A_ret
- .space 0x800000
+ .space 0x3800000
.size A, .-A
- .globl pad_hot_0
- .type pad_hot_0, %function
-pad_hot_0:
-.pad_hot_0_entry:
-# FDATA: 1 pad_hot_0 #.pad_hot_0_entry# 100
- ret
- .space 0x3200000
- .size pad_hot_0, .-pad_hot_0
-
.globl B
.type B, %function
B:
.B_entry:
# FDATA: 1 B #.B_entry# 100
+ bl C
cbz x0, .B_ret
b .B_cold
.B_ret:
@@ -99,19 +102,12 @@ B:
ret
.B_cold:
mov x0, #2
+ bl B
+ bl D
b .B_ret
- .space 0x800000
+ .space 0x3800000
.size B, .-B
- .globl pad_hot_1
- .type pad_hot_1, %function
-pad_hot_1:
-.pad_hot_1_entry:
-# FDATA: 1 pad_hot_1 #.pad_hot_1_entry# 100
- ret
- .space 0x3200000
- .size pad_hot_1, .-pad_hot_1
-
.globl C
.type C, %function
C:
@@ -125,24 +121,17 @@ C:
ret
.C_cold:
mov x0, #3
+ bl B
b .C_ret
- .space 0x800000
+ .space 0x3800000
.size C, .-C
- .globl pad_hot_2
- .type pad_hot_2, %function
-pad_hot_2:
-.pad_hot_2_entry:
-# FDATA: 1 pad_hot_2 #.pad_hot_2_entry# 100
- ret
- .space 0x3200000
- .size pad_hot_2, .-pad_hot_2
-
.globl D
.type D, %function
D:
.D_entry:
# FDATA: 1 D #.D_entry# 100
+ bl A
cbz x0, .D_ret
b .D_cold
.D_ret:
@@ -150,24 +139,19 @@ D:
ret
.D_cold:
mov x0, #4
+ bl B
+ bl D
b .D_ret
- .space 0x800000
+ .space 0x3800000
.size D, .-D
- .globl pad_hot_3
- .type pad_hot_3, %function
-pad_hot_3:
-.pad_hot_3_entry:
-# FDATA: 1 pad_hot_3 #.pad_hot_3_entry# 100
- ret
- .space 0x3200000
- .size pad_hot_3, .-pad_hot_3
-
## Force relocation mode.
.reloc 0, R_AARCH64_NONE
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+
+# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
# CHECK-OUTPUT: <__AArch64Thunk_C>:
@@ -179,23 +163,71 @@ pad_hot_3:
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
-# CHECK-OUTPUT: <__AArch64ADRPThunk_A>:
-# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}} <A>
-# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, #0x0
-# CHECK-OUTPUT-NEXT: {{.*}} br x16
+# CHECK-OUTPUT: <D>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+
+# CHECK-OUTPUT: <__AArch64Thunk_D>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
# CHECK-OUTPUT: <A.cold.0>:
-# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x1
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_A>
+
+# CHECK-OUTPUT: <B.cold.0>:
+# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_D>
+
+# CHECK-OUTPUT: <__AArch64ADRPThunk_B>:
+# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-OUTPUT: <__AArch64ADRPThunk_D>:
+# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-OUTPUT: <C.cold.0>:
+# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x3
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
+
+# CHECK-OUTPUT: <D.cold.0>:
+# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x4
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
# CHECK-HFE-OUTPUT: <A.cold.0>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
+
+# CHECK-HFE-OUTPUT: <B.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x2
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
+
+# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_D>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_B>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-HFE-OUTPUT: <C.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x3
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_B>
+
+# CHECK-HFE-OUTPUT: <D.cold.0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
+
+# CHECK-HFE-OUTPUT: <__AArch64Thunk_B>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <B>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+
+# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
# CHECK-HFE-OUTPUT: <__AArch64Thunk_C>:
@@ -206,3 +238,6 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+
+# CHECK-HFE-OUTPUT: <D>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
>From ce5c42e7c17d68de036853b093d0885fe44e74be Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 3 Sep 2026 13:07:21 +0000
Subject: [PATCH 05/22] start from hot functions when building clusters
---
bolt/lib/Passes/LongJmp.cpp | 73 +++++++++---
.../AArch64/relax-branches-with-thunk-chain.s | 105 ++++++++++++------
.../test/AArch64/relax-cross-fragment-calls.s | 46 ++++++--
3 files changed, 163 insertions(+), 61 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 71ff27ccc4d0e..1219a3accd9a4 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -17,6 +17,7 @@
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
+#include <algorithm>
#define DEBUG_TYPE "longjmp"
@@ -1052,6 +1053,12 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
const FunctionFragment *FF;
size_t FunctionIndex;
SmallString<32> SectionName;
+ uint64_t Size;
+ };
+
+ struct FragmentRange {
+ size_t Begin;
+ size_t End;
};
FragmentClusterLayout Layout;
@@ -1066,8 +1073,9 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
if (FF.empty() && !BF->hasConstantIsland())
continue;
- OrderedFragments.push_back(
- {&FF, I, BF->getCodeSectionName(FF.getFragmentNum())});
+ OrderedFragments.push_back({&FF, I,
+ BF->getCodeSectionName(FF.getFragmentNum()),
+ estimateFragmentSize(*BF, FF)});
}
}
@@ -1078,25 +1086,57 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
return BC.compareSectionNames(A.SectionName, B.SectionName);
});
- auto addFragmentToCluster = [&](const OutputFragment &Fragment) {
+ auto buildClusterRanges = [&]() {
+ SmallVector<FragmentRange> ClusterRanges;
+ if (OrderedFragments.empty())
+ return ClusterRanges;
+
+ if (!opts::HotFunctionsAtEnd) {
+ size_t Begin = 0;
+ uint64_t Size = OrderedFragments[0].Size;
+ for (size_t I = 1; I < OrderedFragments.size(); ++I) {
+ if (Size + OrderedFragments[I].Size > opts::MaxClusterSize) {
+ ClusterRanges.push_back({Begin, I});
+ Begin = I;
+ Size = 0;
+ }
+ Size += OrderedFragments[I].Size;
+ }
+ ClusterRanges.push_back({Begin, OrderedFragments.size()});
+ return ClusterRanges;
+ }
+
+ size_t End = OrderedFragments.size();
+ uint64_t Size = OrderedFragments.back().Size;
+ for (size_t I = End - 1; I > 0;) {
+ --I;
+ if (Size + OrderedFragments[I].Size > opts::MaxClusterSize) {
+ ClusterRanges.push_back({I + 1, End});
+ End = I + 1;
+ Size = 0;
+ }
+ Size += OrderedFragments[I].Size;
+ }
+ ClusterRanges.push_back({0, End});
+ std::reverse(ClusterRanges.begin(), ClusterRanges.end());
+ return ClusterRanges;
+ };
+
+ auto addFragmentToCluster = [&](const OutputFragment &Fragment,
+ FragmentCluster &FC,
+ const unsigned ClusterNum) {
BinaryFunction &BF = *OutputFunctions[Fragment.FunctionIndex];
const FunctionFragment &FF = *Fragment.FF;
- const uint64_t FFSize = estimateFragmentSize(BF, FF);
- if (Layout.Clusters.empty() ||
- Layout.Clusters.back().Size + FFSize > opts::MaxClusterSize) {
- Layout.Clusters.emplace_back(FragmentCluster());
- FragmentCluster &FC = Layout.Clusters.back();
+ if (FC.NumFragments == 0) {
FC.StartSectionName = Fragment.SectionName;
FC.FirstFunctionIndex = Fragment.FunctionIndex;
}
- FragmentCluster &FC = Layout.Clusters.back();
FC.EndSectionName = Fragment.SectionName;
FC.LastFunctionIndex = Fragment.FunctionIndex;
++FC.NumFragments;
- EstimatedSize += FFSize;
- const unsigned ClusterNum = Layout.Clusters.size() - 1;
+ EstimatedSize += Fragment.Size;
// Map primary entry points.
if (FF.isMainFragment())
@@ -1113,11 +1153,16 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
Layout.SymToCluster[EntrySymbol] = ClusterNum;
}
- FC.Size += FFSize;
+ FC.Size += Fragment.Size;
};
- for (const OutputFragment &Fragment : OrderedFragments)
- addFragmentToCluster(Fragment);
+ for (const FragmentRange &Range : buildClusterRanges()) {
+ Layout.Clusters.emplace_back(FragmentCluster());
+ FragmentCluster &FC = Layout.Clusters.back();
+ const unsigned ClusterNum = Layout.Clusters.size() - 1;
+ for (size_t I = Range.Begin; I < Range.End; ++I)
+ addFragmentToCluster(OrderedFragments[I], FC, ClusterNum);
+ }
if (Layout.Clusters.empty())
return Layout;
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index 5b2176d70fe33..977e9781c5640 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -6,16 +6,17 @@
## --hot-functions-at-end. With the default 124MiB function-fragment cluster size,
## branch relaxation sees:
##
-## normal layout:
-## cluster 0, .text: A, pad_hot_0, B, pad_hot_1
-## cluster 1, .text: C, pad_hot_2, D, pad_hot_3
-## cluster 2, .text.cold: A.cold, B.cold, C.cold, D.cold
+## normal layout
+## -------------
+## cluster 0 (~112MiB), .text: A, pad_hot_0, B, pad_hot_1
+## cluster 1 (~112MiB), .text: C, pad_hot_2, D, pad_hot_3
+## cluster 2 (~64MiB), .text.cold: A.cold, B.cold, C.cold, D.cold
##
-## --hot-functions-at-end:
-## cluster 0: .text.cold A.cold, B.cold, C.cold, D.cold
-## .text A, pad_hot_0
-## cluster 1: .text B, pad_hot_1, C, pad_hot_2
-## cluster 2: .text D, pad_hot_3
+## hot-functions-at-end
+## --------------------
+## cluster 0 (~64MiB), .text.cold: A.cold, B.cold, C.cold, D.cold
+## cluster 1 (~112MiB), .text: A, pad_hot_0, B, pad_hot_1
+## cluster 2 (~112MiB), .text: C, pad_hot_2, D, pad_hot_3
# REQUIRES: system-linux
@@ -33,17 +34,35 @@
# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_4,__AArch64BranchForwardThunk_5,__AArch64BranchForwardThunk_8,__AArch64BranchForwardThunk_10,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_3,__AArch64BranchBackwardThunk_6,__AArch64BranchBackwardThunk_7,__AArch64BranchBackwardThunk_9,__AArch64BranchBackwardThunk_11 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchForwardThunk_10,__AArch64BranchForwardThunk_11,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5,__AArch64BranchBackwardThunk_8,__AArch64BranchBackwardThunk_9 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
+# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440604 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
+# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440584 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
+# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes
# CHECK-BOLT: BOLT-INFO: relaxed 9 cross-cluster branches
# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 7 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 8 branch thunks created
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 67108944 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440604 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 9 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 1 branch thunks reused
# CHECK-SECTIONS: .text
@@ -240,64 +259,76 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
-
-# CHECK-HFE-OUTPUT: Disassembly of section .text:
-
-# CHECK-HFE-OUTPUT: <A>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_11>
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x10>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_FW]]: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x10>
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_7>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_11>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+
+# CHECK-HFE-OUTPUT: Disassembly of section .text:
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_4>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_8>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
+# CHECK-HFE-OUTPUT: <A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET:[0-9a-f]+]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_BW]] <__AArch64BranchBackwardThunk_0>
+
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[HFE_B_ALT:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[HFE_B_RET]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_0>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_2>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_2>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT: <C>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW]] <__AArch64BranchBackwardThunk_2>
-
# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_6>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_10>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_4>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64BranchBackwardThunk_4>
+
+# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_9>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_8>
+
+# CHECK-HFE-OUTPUT: <C>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64BranchBackwardThunk_5>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_5>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_9>
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 6739cd1437049..b893279c6636a 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -17,17 +17,19 @@
## B.cold -> D HFE: forward long thunk
## D.cold -> D HFE: shares forward long thunk to D
##
-## normal layout:
-## cluster 0, .text: A, B
-## cluster 1, .text: C, D
-## cluster 2, .text.cold: A.cold, B.cold
-## cluster 3, .text.cold: C.cold, D.cold
+## normal layout
+## -------------
+## cluster 0 (~112MiB), .text: A, B
+## cluster 1 (~112MiB), .text: C, D
+## cluster 2 (~112MiB), .text.cold: A.cold, B.cold
+## cluster 3 (~112MiB), .text.cold: C.cold, D.cold
##
-## --hot-functions-at-end:
-## cluster 0: .text.cold A.cold, B.cold
-## cluster 1: .text.cold C.cold, D.cold
-## cluster 2: .text A, B
-## cluster 3: .text C, D
+## hot-functions-at-end
+## --------------------
+## cluster 0 (~112MiB), .text.cold: A.cold, B.cold
+## cluster 1 (~112MiB), .text.cold: C.cold, D.cold
+## cluster 2 (~112MiB), .text: A, B
+## cluster 3 (~112MiB), .text: C, D
# REQUIRES: system-linux
@@ -49,6 +51,18 @@
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
+# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
+# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
+# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440568 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
+# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
# CHECK-BOLT: BOLT-INFO: relaxed 4 adjacent cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: 3 short thunks created
@@ -59,6 +73,18 @@
# CHECK-BOLT: BOLT-INFO: 16 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440568 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 5 adjacent cluster calls with thunks
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 remote cluster calls with thunks
# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
>From cbb7b4e495495f1ed7f4cca73629d312bcdf60ee Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 3 Sep 2026 14:19:44 +0000
Subject: [PATCH 06/22] rename thunks as
__AArch64_<direction>_<kind>_<target-or-id>
---
bolt/lib/Passes/LongJmp.cpp | 22 ++--
.../AArch64/relax-branches-with-thunk-chain.s | 104 +++++++++---------
bolt/test/AArch64/relax-calls.s | 4 +-
.../test/AArch64/relax-cross-fragment-calls.s | 56 +++++-----
4 files changed, 95 insertions(+), 91 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 1219a3accd9a4..40cfc0d430006 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1269,19 +1269,22 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
size_t NumLongThunks = 0;
size_t NumShortThunksReused = 0;
size_t NumLongThunksReused = 0;
- auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort) {
+ auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort,
+ bool IsForward) {
BinaryFunction *Thunk = nullptr;
+ std::string ThunkName =
+ (Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
+ (IsShort ? "short_call_" : "long_call_") + TargetSymbol->getName())
+ .str();
if (IsShort) {
++NumShortThunks;
- Thunk = BC.createThunkBinaryFunction("__AArch64Thunk_" +
- TargetSymbol->getName().str());
+ Thunk = BC.createThunkBinaryFunction(ThunkName);
MCInst Inst;
BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
Thunk->addBasicBlock()->addInstruction(Inst);
} else {
++NumLongThunks;
- Thunk = BC.createThunkBinaryFunction("__AArch64ADRPThunk_" +
- TargetSymbol->getName().str());
+ Thunk = BC.createThunkBinaryFunction(ThunkName);
InstructionListType Instructions;
BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
Thunk->addBasicBlock()->addInstructions(Instructions);
@@ -1337,7 +1340,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
}
}
- BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort);
+ BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort, IsForward);
FragmentCluster &ThunkCluster = Clusters[SourceCluster];
Thunk->setCodeSectionName(ThunkCluster.getThunkSectionName(IsForward));
@@ -1445,9 +1448,10 @@ void LongJmpPass::relaxUnconditionalBranches(
size_t NumBranchThunksReused = 0;
auto createBranchThunk = [&](const MCSymbol *TargetSymbol,
const bool IsForward) {
- std::string ThunkName = IsForward ? "__AArch64BranchForwardThunk_"
- : "__AArch64BranchBackwardThunk_";
- ThunkName += std::to_string(NumBranchThunks++);
+ std::string ThunkName =
+ (Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
+ "branch_chain_" + Twine(NumBranchThunks++))
+ .str();
BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(ThunkName);
MCInst Inst;
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index 977e9781c5640..e6a676ebc8d7b 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -31,10 +31,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_4,__AArch64BranchForwardThunk_5,__AArch64BranchForwardThunk_8,__AArch64BranchForwardThunk_10,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_3,__AArch64BranchBackwardThunk_6,__AArch64BranchBackwardThunk_7,__AArch64BranchBackwardThunk_9,__AArch64BranchBackwardThunk_11 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_forward_branch_chain_8,__AArch64_forward_branch_chain_10,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3,__AArch64_backward_branch_chain_6,__AArch64_backward_branch_chain_7,__AArch64_backward_branch_chain_9,__AArch64_backward_branch_chain_11 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64BranchForwardThunk_1,__AArch64BranchForwardThunk_3,__AArch64BranchForwardThunk_6,__AArch64BranchForwardThunk_7,__AArch64BranchForwardThunk_10,__AArch64BranchForwardThunk_11,__AArch64BranchBackwardThunk_0,__AArch64BranchBackwardThunk_2,__AArch64BranchBackwardThunk_4,__AArch64BranchBackwardThunk_5,__AArch64BranchBackwardThunk_8,__AArch64BranchBackwardThunk_9 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_forward_branch_chain_7,__AArch64_forward_branch_chain_10,__AArch64_forward_branch_chain_11,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5,__AArch64_backward_branch_chain_8,__AArch64_backward_branch_chain_9 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -181,154 +181,154 @@ pad_hot_3:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[B_ALT:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[B_RET:[0-9a-f]+]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_5>
-# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64BranchForwardThunk_5>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_5>
+# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64_forward_branch_chain_5>
# CHECK-OUTPUT-NEXT: [[B_RET]]: {{.*}} ret
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_1>:
-# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_0>
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_1>:
+# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_5>:
-# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_4>
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_5>:
+# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_4>
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_2>:
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_2>:
# CHECK-OUTPUT-NEXT: [[A_BW1:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_6>:
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_6>:
# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x10>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[C_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_8>
+# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_8>
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_10>
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_0>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_0>:
# CHECK-OUTPUT-NEXT: [[A_FW1]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_4>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_4>:
# CHECK-OUTPUT-NEXT: [[B_FW1]]: {{.*}} b 0x[[B_COLD:[0-9a-f]+]] <B.cold.0>
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_8>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_8>:
# CHECK-OUTPUT-NEXT: [[C_FW0]]: {{.*}} b 0x[[C_COLD:[0-9a-f]+]] <C.cold.0>
-# CHECK-OUTPUT: <__AArch64BranchForwardThunk_10>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_10>:
# CHECK-OUTPUT-NEXT: [[D_FW0]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
# CHECK-OUTPUT: Disassembly of section .text.cold:
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_3>:
-# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_BW1]] <__AArch64BranchBackwardThunk_2>
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_3>:
+# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_BW1]] <__AArch64_backward_branch_chain_2>
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_7>:
-# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64BranchBackwardThunk_6>
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_7>:
+# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64_backward_branch_chain_6>
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_9>:
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_9>:
# CHECK-OUTPUT-NEXT: [[C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[C_RET]] <C+0x4>
-# CHECK-OUTPUT: <__AArch64BranchBackwardThunk_11>:
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_11>:
# CHECK-OUTPUT-NEXT: [[D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[D_RET]] <D+0x4>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT-NEXT: [[A_COLD]]: {{.*}} mov x0, #0x1
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64BranchBackwardThunk_3>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_branch_chain_3>
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: [[B_COLD]]: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64BranchBackwardThunk_7>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64_backward_branch_chain_7>
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: [[C_COLD]]: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64BranchBackwardThunk_9>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64_backward_branch_chain_9>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: [[D_COLD]]: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[D_BW0]] <__AArch64BranchBackwardThunk_11>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[D_BW0]] <__AArch64_backward_branch_chain_11>
# CHECK-HFE-OUTPUT: Disassembly of section .text.cold:
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64BranchForwardThunk_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_7>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_7>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64BranchForwardThunk_11>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_11>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_1>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_1>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_FW]]: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_3>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_3>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x10>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_7>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_6>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_7>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_11>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64BranchForwardThunk_10>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_11>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_10>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_0>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_2>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_2>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_4>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_4>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_8>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_8>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_BW]] <__AArch64BranchBackwardThunk_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_BW]] <__AArch64_backward_branch_chain_0>
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[HFE_B_ALT:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[HFE_B_RET]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_2>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64BranchBackwardThunk_2>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64_backward_branch_chain_2>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64_backward_branch_chain_2>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_6>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_6>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchForwardThunk_10>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_10>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64BranchBackwardThunk_4>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_4>
-# CHECK-HFE-OUTPUT: <__AArch64BranchBackwardThunk_9>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64BranchBackwardThunk_8>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_9>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_8>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64BranchBackwardThunk_5>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64_backward_branch_chain_5>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64BranchBackwardThunk_9>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_9>
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index f6394b7a39df7..94bca7309290a 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -63,8 +63,8 @@ hot:
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64ADRPThunk_foo>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64Thunk_bar>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_long_call_foo>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_short_call_bar>
# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
.global _start
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index b893279c6636a..e04194a8c2f3f 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -44,10 +44,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_C,__AArch64Thunk_D,__AArch64ADRPThunk_B,__AArch64ADRPThunk_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_C,__AArch64_backward_short_call_A,__AArch64_backward_short_call_D,__AArch64_backward_long_call_B,__AArch64_backward_long_call_D \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64Thunk_A,__AArch64Thunk_B,__AArch64Thunk_C,__AArch64ADRPThunk_B,__AArch64ADRPThunk_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_B,__AArch64_forward_short_call_C,__AArch64_backward_short_call_A,__AArch64_forward_long_call_B,__AArch64_forward_long_call_D \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
@@ -178,92 +178,92 @@ D:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
# CHECK-OUTPUT: <B>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_C>
-# CHECK-OUTPUT: <__AArch64Thunk_C>:
+# CHECK-OUTPUT: <__AArch64_forward_short_call_C>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
-# CHECK-OUTPUT: <__AArch64Thunk_A>:
+# CHECK-OUTPUT: <__AArch64_backward_short_call_A>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-OUTPUT: <C>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
# CHECK-OUTPUT: <D>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
-# CHECK-OUTPUT: <__AArch64Thunk_D>:
+# CHECK-OUTPUT: <__AArch64_backward_short_call_D>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_D>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_D>
-# CHECK-OUTPUT: <__AArch64ADRPThunk_B>:
+# CHECK-OUTPUT: <__AArch64_backward_long_call_B>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
-# CHECK-OUTPUT: <__AArch64ADRPThunk_D>:
+# CHECK-OUTPUT: <__AArch64_backward_long_call_D>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_D>
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_B>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
-# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_D>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
-# CHECK-HFE-OUTPUT: <__AArch64ADRPThunk_B>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_B>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_B>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64ADRPThunk_D>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
-# CHECK-HFE-OUTPUT: <__AArch64Thunk_B>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <B>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
# CHECK-HFE-OUTPUT: <B>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_C>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_C>
-# CHECK-HFE-OUTPUT: <__AArch64Thunk_C>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
-# CHECK-HFE-OUTPUT: <__AArch64Thunk_A>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_short_call_A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-HFE-OUTPUT: <C>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
# CHECK-HFE-OUTPUT: <D>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64Thunk_A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
>From a1933c1ce77104ad60f9f1027aa33e0011aaf475 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 3 Sep 2026 15:46:43 +0000
Subject: [PATCH 07/22] Do not assume PLT proximity to hot text by default. The
prebolted binary may have a big hole between PLT and the location where the
new text will be emitted
---
bolt/lib/Passes/LongJmp.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 40cfc0d430006..1146d5f550724 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -40,7 +40,7 @@ static cl::opt<bool>
static cl::opt<bool> RelaxPLT("relax-plt",
cl::desc("indicate PLT proximity to hot text"),
- cl::init(true), cl::cat(BoltOptCategory));
+ cl::init(false), cl::cat(BoltOptCategory));
static cl::opt<unsigned long long> MaxClusterSize(
"max-cluster-size",
>From 119e2356053c41479018d11cc8771b3117be404c Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sat, 5 Sep 2026 08:51:42 +0000
Subject: [PATCH 08/22] Use layout offsets to skip redundant in range branch
relaxation
---
bolt/include/bolt/Passes/LongJmp.h | 2 +
bolt/lib/Passes/LongJmp.cpp | 70 +++++++++++---
.../AArch64/relax-branches-with-thunk-chain.s | 84 +++++++---------
bolt/test/AArch64/relax-calls.s | 8 +-
.../test/AArch64/relax-cross-fragment-calls.s | 95 +++++++++----------
5 files changed, 143 insertions(+), 116 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index e021e8fad503b..286910273ffd5 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -127,7 +127,9 @@ class LongJmpPass : public BinaryFunctionPass {
struct FragmentClusterLayout {
SmallVector<FragmentCluster, 4> Clusters;
DenseMap<const BinaryBasicBlock *, unsigned> BBToCluster;
+ DenseMap<const BinaryBasicBlock *, uint64_t> BBToOffset;
DenseMap<const MCSymbol *, unsigned> SymToCluster;
+ DenseMap<const MCSymbol *, uint64_t> SymToOffset;
};
FragmentClusterLayout
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 1146d5f550724..1a4f253cf65e4 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -73,6 +73,13 @@ static void relaxStubToLongJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
BC.MIB->applyBTIFixupToTarget(StubBB);
}
+static bool isWithinClusterRange(uint64_t SourceOffset, uint64_t TargetOffset) {
+ const uint64_t Distance = SourceOffset <= TargetOffset
+ ? TargetOffset - SourceOffset
+ : SourceOffset - TargetOffset;
+ return Distance < opts::MaxClusterSize;
+}
+
static BinaryBasicBlock *getBBAtHotColdSplitPoint(BinaryFunction &Func) {
if (!Func.isSplit() || Func.empty())
return nullptr;
@@ -1063,7 +1070,6 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
FragmentClusterLayout Layout;
SmallVector<OutputFragment> OrderedFragments;
- uint64_t EstimatedSize = 0;
for (size_t I = 0; I < OutputFunctions.size(); ++I) {
BinaryFunction *BF = OutputFunctions[I];
if (!BC.shouldEmit(*BF) || BF->isPatch())
@@ -1122,11 +1128,13 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
return ClusterRanges;
};
- auto addFragmentToCluster = [&](const OutputFragment &Fragment,
- FragmentCluster &FC,
- const unsigned ClusterNum) {
+ uint64_t LayoutOffset = 0;
+ auto addFragmentToCluster = [&](const OutputFragment &Fragment) {
+ FragmentCluster &FC = Layout.Clusters.back();
+ const unsigned ClusterNum = Layout.Clusters.size() - 1;
BinaryFunction &BF = *OutputFunctions[Fragment.FunctionIndex];
const FunctionFragment &FF = *Fragment.FF;
+ const uint64_t FragmentOffset = LayoutOffset;
if (FC.NumFragments == 0) {
FC.StartSectionName = Fragment.SectionName;
@@ -1136,38 +1144,46 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
FC.EndSectionName = Fragment.SectionName;
FC.LastFunctionIndex = Fragment.FunctionIndex;
++FC.NumFragments;
- EstimatedSize += Fragment.Size;
// Map primary entry points.
if (FF.isMainFragment())
- for (const MCSymbol *Symbol : BF.getSymbols())
+ for (const MCSymbol *Symbol : BF.getSymbols()) {
Layout.SymToCluster[Symbol] = ClusterNum;
+ Layout.SymToOffset[Symbol] = FragmentOffset;
+ }
+ uint64_t BBOffset = FragmentOffset;
for (const BinaryBasicBlock *BB : FF) {
Layout.BBToCluster[BB] = ClusterNum;
- if (const MCSymbol *Label = BB->getLabel())
+ Layout.BBToOffset[BB] = BBOffset;
+ if (const MCSymbol *Label = BB->getLabel()) {
Layout.SymToCluster[Label] = ClusterNum;
+ Layout.SymToOffset[Label] = BBOffset;
+ }
// Map secondary entry points.
- if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
+ if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB)) {
Layout.SymToCluster[EntrySymbol] = ClusterNum;
+ Layout.SymToOffset[EntrySymbol] = BBOffset;
+ }
+
+ BBOffset += BB->estimateSize();
}
FC.Size += Fragment.Size;
+ LayoutOffset += Fragment.Size;
};
for (const FragmentRange &Range : buildClusterRanges()) {
- Layout.Clusters.emplace_back(FragmentCluster());
- FragmentCluster &FC = Layout.Clusters.back();
- const unsigned ClusterNum = Layout.Clusters.size() - 1;
+ Layout.Clusters.emplace_back();
for (size_t I = Range.Begin; I < Range.End; ++I)
- addFragmentToCluster(OrderedFragments[I], FC, ClusterNum);
+ addFragmentToCluster(OrderedFragments[I]);
}
if (Layout.Clusters.empty())
return Layout;
- LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << EstimatedSize
+ LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << LayoutOffset
<< '\n');
// Print cluster stats.
@@ -1205,7 +1221,9 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
FragmentClusterLayout &Layout) {
auto &Clusters = Layout.Clusters;
auto &BBToCluster = Layout.BBToCluster;
+ auto &BBToOffset = Layout.BBToOffset;
auto &SymToCluster = Layout.SymToCluster;
+ auto &SymToOffset = Layout.SymToOffset;
struct CrossClusterCall {
MCInst *Inst;
@@ -1231,8 +1249,13 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
if (SourceIt == BBToCluster.end())
continue;
const unsigned SourceCluster = SourceIt->second;
+ // The cluster lookup above guarantees this BB has a layout offset.
+ uint64_t InstOffset = BBToOffset[&BB];
for (MCInst &Inst : BB) {
+ const uint64_t SourceOffset = InstOffset;
+ InstOffset += BC.computeInstructionSize(Inst);
+
if (!BC.MIB->isCall(Inst) && !BC.MIB->isUnconditionalBranch(Inst))
continue;
@@ -1251,11 +1274,18 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
!BC.getSymbolValue(*TargetSymbol))
continue;
- // If not found TargetCluster becomes UINT_MAX.
+ // If not found use -1 so the computed distance is out of range.
unsigned TargetCluster = Found ? It->second : -1;
if (TargetCluster == SourceCluster)
continue;
+ // A target with a cluster was mapped by the layout builder, so it
+ // also has an offset. Otherwise use -1 so the computed distance is
+ // conservatively treated as out of range.
+ const uint64_t TargetOffset = Found ? SymToOffset[TargetSymbol] : -1;
+ if (isWithinClusterRange(SourceOffset, TargetOffset))
+ continue;
+
CrossClusterCall Call{&Inst, TargetSymbol, SourceCluster,
TargetCluster};
auto &Calls = clusterDistance(Call) == 1 ? AdjacentClusterCalls
@@ -1402,7 +1432,9 @@ void LongJmpPass::relaxUnconditionalBranches(
FragmentClusterLayout &Layout) {
auto &Clusters = Layout.Clusters;
auto &BBToCluster = Layout.BBToCluster;
+ auto &BBToOffset = Layout.BBToOffset;
auto &SymToCluster = Layout.SymToCluster;
+ auto &SymToOffset = Layout.SymToOffset;
struct CrossClusterBranch {
MCInst *Inst;
@@ -1421,8 +1453,13 @@ void LongJmpPass::relaxUnconditionalBranches(
if (SourceIt == BBToCluster.end())
continue;
const unsigned SourceCluster = SourceIt->second;
+ // The cluster lookup above guarantees this BB has a layout offset.
+ uint64_t InstOffset = BBToOffset[&BB];
for (MCInst &Inst : BB) {
+ const uint64_t SourceOffset = InstOffset;
+ InstOffset += BC.computeInstructionSize(Inst);
+
if (!BC.MIB->isUnconditionalBranch(Inst))
continue;
@@ -1438,6 +1475,11 @@ void LongJmpPass::relaxUnconditionalBranches(
if (SourceCluster == TargetCluster)
continue;
+ // The cluster lookup above guarantees this symbol has a layout offset.
+ const uint64_t TargetOffset = SymToOffset[TargetSymbol];
+ if (isWithinClusterRange(SourceOffset, TargetOffset))
+ continue;
+
CrossClusterBranches.push_back(
{&Inst, TargetSymbol, SourceCluster, TargetCluster});
}
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index e6a676ebc8d7b..5626eb6e90bbc 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -1,10 +1,9 @@
## Relax unconditional branches using thunk chains across fragment clusters.
## A/B/C/D contain 16MiB islands and are interleaved with 40MiB hot pad functions.
-## This deliberately tests that hot and cold fragments may coexist in a cluster
-## when --hot-functions-at-end is specified. B has duplicate hot-to-cold branches
-## which share a forward chain in the normal layout and a backward chain with
-## --hot-functions-at-end. With the default 124MiB function-fragment cluster size,
-## branch relaxation sees:
+## B has duplicate hot-to-cold branches which share a forward chain in the
+## normal layout. With --hot-functions-at-end, A/B hot-to-cold branches are
+## close enough to remain direct, while C/D still need backward chains. With the
+## default 124MiB function-fragment cluster size, branch relaxation sees:
##
## normal layout
## -------------
@@ -31,10 +30,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_forward_branch_chain_8,__AArch64_forward_branch_chain_10,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3,__AArch64_backward_branch_chain_6,__AArch64_backward_branch_chain_7,__AArch64_backward_branch_chain_9,__AArch64_backward_branch_chain_11 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_forward_branch_chain_8,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3,__AArch64_backward_branch_chain_6,__AArch64_backward_branch_chain_7,__AArch64_backward_branch_chain_9 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_forward_branch_chain_7,__AArch64_forward_branch_chain_10,__AArch64_forward_branch_chain_11,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5,__AArch64_backward_branch_chain_8,__AArch64_backward_branch_chain_9 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_2,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_forward_branch_chain_7,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_1,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -47,8 +46,8 @@
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes
-# CHECK-BOLT: BOLT-INFO: relaxed 9 cross-cluster branches
-# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
+# CHECK-BOLT: BOLT-INFO: relaxed 7 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: 10 branch thunks created
# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
@@ -61,9 +60,8 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 9 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: 1 branch thunks reused
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: 8 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -210,7 +208,7 @@ pad_hot_3:
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_10>
+# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
# CHECK-OUTPUT: <__AArch64_forward_branch_chain_0>:
# CHECK-OUTPUT-NEXT: [[A_FW1]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
@@ -221,9 +219,6 @@ pad_hot_3:
# CHECK-OUTPUT: <__AArch64_forward_branch_chain_8>:
# CHECK-OUTPUT-NEXT: [[C_FW0]]: {{.*}} b 0x[[C_COLD:[0-9a-f]+]] <C.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_10>:
-# CHECK-OUTPUT-NEXT: [[D_FW0]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
-
# CHECK-OUTPUT: Disassembly of section .text.cold:
# CHECK-OUTPUT: <__AArch64_backward_branch_chain_3>:
@@ -235,9 +230,6 @@ pad_hot_3:
# CHECK-OUTPUT: <__AArch64_backward_branch_chain_9>:
# CHECK-OUTPUT-NEXT: [[C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[C_RET]] <C+0x4>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_11>:
-# CHECK-OUTPUT-NEXT: [[D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[D_RET]] <D+0x4>
-
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT-NEXT: [[A_COLD]]: {{.*}} mov x0, #0x1
# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_branch_chain_3>
@@ -252,83 +244,71 @@ pad_hot_3:
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: [[D_COLD]]: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[D_BW0]] <__AArch64_backward_branch_chain_11>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[D_RET]] <D+0x4>
# CHECK-HFE-OUTPUT: Disassembly of section .text.cold:
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x1
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_A_FW:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x{{[0-9a-f]+}} <A+0x4>
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_FW:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x{{[0-9a-f]+}} <B+0x10>
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_7>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_11>
-
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_1>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_FW]]: {{.*}} b 0x[[HFE_A_RET:[0-9a-f]+]] <A+0x4>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_7>
# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_FW]]: {{.*}} b 0x[[HFE_B_RET:[0-9a-f]+]] <B+0x10>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_2>
# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_7>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
-
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_11>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_10>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_0>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
-
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_2>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_BW:[0-9a-f]+]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
-
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_4>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_8>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_4>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_A_BR:[0-9a-f]+]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_BW]] <__AArch64_backward_branch_chain_0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} ret
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_A_BR]]: {{.*}} b 0x[[HFE_A_COLD]] <A.cold.0>
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[HFE_B_ALT:[0-9a-f]+]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[HFE_B_RET]] <{{.*}}>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_BW]] <__AArch64_backward_branch_chain_2>
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_BW]] <__AArch64_backward_branch_chain_2>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[HFE_B_RET:[0-9a-f]+]] <{{.*}}>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_6>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_2>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_10>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_6>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_4>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_1>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_0>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_9>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_8>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_4>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64_backward_branch_chain_5>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64_backward_branch_chain_1>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_9>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_5>
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 94bca7309290a..4c33e964683ad 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -55,11 +55,15 @@ hot:
# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
## Check the number of thunks created in other modes.
-# CHECK-BOLT: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: 3 long thunks created
+# CHECK-BOLT: BOLT-INFO: 1 long thunks reused
-# CHECK-BOLT-HOT-END: BOLT-INFO: 4 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 1 adjacent cluster calls with thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 3 remote cluster calls with thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: 1 short thunks created
# CHECK-BOLT-HOT-END: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 1 long thunks reused
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index e04194a8c2f3f..7a646059df429 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -1,14 +1,15 @@
## Relax direct calls using thunks across function fragment clusters. A/B/C/D
-## contain 56MiB islands. With --split-functions, BOLT places the cold blocks
-## of A/B/C/D in .text.cold. The test exercises forward and backward short
-## thunks for adjacent clusters, as well as long thunk reuse across remote
-## clusters. With the default 124MiB function-fragment cluster size, call
-## relaxation sees:
+## contain 42MiB islands. With --split-functions, BOLT places the cold blocks
+## of A/B/C/D in .text.cold. The test exercises direct adjacent-cluster calls,
+## forward and backward short thunks for adjacent clusters, and long thunk reuse
+## across remote clusters. With the default 124MiB function-fragment cluster
+## size, call relaxation sees:
##
## A -> B same cluster, direct
-## B -> C forward short thunk
-## C -> A backward short thunk
-## D -> A shares backward short thunk to A
+## B -> C adjacent cluster but close enough, direct
+## C -> A adjacent cluster but close enough, direct
+## A -> D forward short thunk
+## D -> A backward short thunk
##
## B.cold -> B normal: backward long thunk
## C.cold -> B normal: shares backward long thunk to B
@@ -19,17 +20,17 @@
##
## normal layout
## -------------
-## cluster 0 (~112MiB), .text: A, B
-## cluster 1 (~112MiB), .text: C, D
-## cluster 2 (~112MiB), .text.cold: A.cold, B.cold
-## cluster 3 (~112MiB), .text.cold: C.cold, D.cold
+## cluster 0 (~84MiB), .text: A, B
+## cluster 1 (~84MiB), .text: C, D
+## cluster 2 (~84MiB), .text.cold: A.cold, B.cold
+## cluster 3 (~84MiB), .text.cold: C.cold, D.cold
##
## hot-functions-at-end
## --------------------
-## cluster 0 (~112MiB), .text.cold: A.cold, B.cold
-## cluster 1 (~112MiB), .text.cold: C.cold, D.cold
-## cluster 2 (~112MiB), .text: A, B
-## cluster 3 (~112MiB), .text: C, D
+## cluster 0 (~84MiB), .text.cold: A.cold, B.cold
+## cluster 1 (~84MiB), .text.cold: C.cold, D.cold
+## cluster 2 (~84MiB), .text: A, B
+## cluster 3 (~84MiB), .text: C, D
# REQUIRES: system-linux
@@ -44,30 +45,29 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_C,__AArch64_backward_short_call_A,__AArch64_backward_short_call_D,__AArch64_backward_long_call_B,__AArch64_backward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_backward_long_call_B,__AArch64_backward_long_call_D \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_B,__AArch64_forward_short_call_C,__AArch64_backward_short_call_A,__AArch64_forward_long_call_B,__AArch64_forward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_B,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_forward_long_call_B,__AArch64_forward_long_call_D \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080456 estimated bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440568 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080440 estimated bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440576 estimated bytes
-# CHECK-BOLT: BOLT-INFO: relaxed 4 adjacent cluster calls with thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT: BOLT-INFO: relaxed 2 adjacent cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: 3 short thunks created
+# CHECK-BOLT: BOLT-INFO: 2 short thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks created
-# CHECK-BOLT: BOLT-INFO: 1 short thunks reused
# CHECK-BOLT: BOLT-INFO: 2 long thunks reused
# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
# CHECK-BOLT: BOLT-INFO: 16 branch thunks created
@@ -75,21 +75,20 @@
# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440568 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080440 estimated bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080456 estimated bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440576 estimated bytes
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 5 adjacent cluster calls with thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 adjacent cluster calls with thunks
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 remote cluster calls with thunks
# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 2 long thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: 2 short thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
# CHECK-BOLT-HFE: BOLT-INFO: 16 branch thunks created
@@ -104,6 +103,7 @@ A:
.A_entry:
# FDATA: 1 A #.A_entry# 100
bl B
+ bl D
cbz x0, .A_ret
b .A_cold
.A_ret:
@@ -112,7 +112,7 @@ A:
.A_cold:
mov x0, #1
b .A_ret
- .space 0x3800000
+ .space 0x2a00000
.size A, .-A
.globl B
@@ -131,7 +131,7 @@ B:
bl B
bl D
b .B_ret
- .space 0x3800000
+ .space 0x2a00000
.size B, .-B
.globl C
@@ -149,7 +149,7 @@ C:
mov x0, #3
bl B
b .C_ret
- .space 0x3800000
+ .space 0x2a00000
.size C, .-C
.globl D
@@ -168,7 +168,7 @@ D:
bl B
bl D
b .D_ret
- .space 0x3800000
+ .space 0x2a00000
.size D, .-D
## Force relocation mode.
@@ -176,31 +176,29 @@ D:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D>
# CHECK-OUTPUT: <B>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_C>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-OUTPUT: <__AArch64_forward_short_call_C>:
-# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
+# CHECK-OUTPUT: <__AArch64_forward_short_call_D>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
# CHECK-OUTPUT: <__AArch64_backward_short_call_A>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-OUTPUT: <C>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
-# CHECK-OUTPUT: <__AArch64_backward_short_call_D>:
-# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
-
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_D>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <D>
# CHECK-OUTPUT: <__AArch64_backward_long_call_B>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
@@ -244,7 +242,7 @@ D:
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_B>:
@@ -252,18 +250,19 @@ D:
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D>
# CHECK-HFE-OUTPUT: <B>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_C>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_C>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <C>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_D>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
# CHECK-HFE-OUTPUT: <__AArch64_backward_short_call_A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-HFE-OUTPUT: <C>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
>From 6ff4accd2b0a5e1a5b231232a563c97a34446c85 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sun, 6 Sep 2026 16:52:24 +0000
Subject: [PATCH 09/22] Select short call thunks over long thunks when possible
---
bolt/include/bolt/Passes/LongJmp.h | 5 ++
bolt/lib/Passes/LongJmp.cpp | 48 ++++++++-----
bolt/test/AArch64/relax-calls.s | 16 +++--
.../test/AArch64/relax-cross-fragment-calls.s | 72 +++++++++----------
4 files changed, 78 insertions(+), 63 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 286910273ffd5..126785f0c2b83 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -95,6 +95,9 @@ class LongJmpPass : public BinaryFunctionPass {
/// Estimated size of the cluster in bytes.
uint64_t Size{0};
+ /// Estimated output offset of the cluster.
+ uint64_t StartOffset{0};
+
/// Number of function fragments in the cluster.
size_t NumFragments{0};
@@ -122,6 +125,8 @@ class LongJmpPass : public BinaryFunctionPass {
StringRef getThunkSectionName(bool IsForward) const {
return IsForward ? EndSectionName : StartSectionName;
}
+
+ uint64_t getEndOffset() const { return StartOffset + Size; }
};
struct FragmentClusterLayout {
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 1a4f253cf65e4..cc53a3cff1708 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1138,6 +1138,7 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
if (FC.NumFragments == 0) {
FC.StartSectionName = Fragment.SectionName;
+ FC.StartOffset = FragmentOffset;
FC.FirstFunctionIndex = Fragment.FunctionIndex;
}
@@ -1228,6 +1229,8 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
struct CrossClusterCall {
MCInst *Inst;
const MCSymbol *TargetSymbol;
+ uint64_t SourceOffset;
+ uint64_t TargetOffset;
unsigned SourceCluster;
unsigned TargetCluster;
};
@@ -1238,8 +1241,20 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
: Call.TargetCluster - Call.SourceCluster;
};
- SmallVector<CrossClusterCall> AdjacentClusterCalls;
- SmallVector<CrossClusterCall> RemoteClusterCalls;
+ auto canUseShortThunk = [&](const CrossClusterCall &Call) {
+ if (Call.TargetCluster == -1u)
+ return false;
+
+ const bool IsForward = Call.SourceCluster < Call.TargetCluster;
+ const FragmentCluster &Source = Clusters[Call.SourceCluster];
+ const uint64_t ThunkOffset =
+ IsForward ? Source.getEndOffset() : Source.StartOffset;
+ return isWithinClusterRange(Call.SourceOffset, ThunkOffset) &&
+ isWithinClusterRange(ThunkOffset, Call.TargetOffset);
+ };
+
+ SmallVector<CrossClusterCall> ShortThunkCalls;
+ SmallVector<CrossClusterCall> LongThunkCalls;
for (BinaryFunction *BF : OutputFunctions) {
if (!BC.shouldEmit(*BF) || BF->isPatch())
continue;
@@ -1275,21 +1290,20 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
continue;
// If not found use -1 so the computed distance is out of range.
- unsigned TargetCluster = Found ? It->second : -1;
+ unsigned TargetCluster = Found ? It->second : -1u;
if (TargetCluster == SourceCluster)
continue;
// A target with a cluster was mapped by the layout builder, so it
// also has an offset. Otherwise use -1 so the computed distance is
// conservatively treated as out of range.
- const uint64_t TargetOffset = Found ? SymToOffset[TargetSymbol] : -1;
+ const uint64_t TargetOffset = Found ? SymToOffset[TargetSymbol] : -1ULL;
if (isWithinClusterRange(SourceOffset, TargetOffset))
continue;
- CrossClusterCall Call{&Inst, TargetSymbol, SourceCluster,
- TargetCluster};
- auto &Calls = clusterDistance(Call) == 1 ? AdjacentClusterCalls
- : RemoteClusterCalls;
+ CrossClusterCall Call{&Inst, TargetSymbol, SourceOffset,
+ TargetOffset, SourceCluster, TargetCluster};
+ auto &Calls = canUseShortThunk(Call) ? ShortThunkCalls : LongThunkCalls;
Calls.push_back(Call);
}
}
@@ -1386,7 +1400,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
// Process farthest calls first so closer remote calls can reuse long thunks
// already placed at adjacent cluster boundaries.
- llvm::stable_sort(RemoteClusterCalls,
+ llvm::stable_sort(LongThunkCalls,
[&](const CrossClusterCall &A, const CrossClusterCall &B) {
return clusterDistance(A) > clusterDistance(B);
});
@@ -1398,19 +1412,19 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
};
- for (CrossClusterCall &Call : AdjacentClusterCalls)
+ for (CrossClusterCall &Call : ShortThunkCalls)
relaxCall(Call, /*IsShort=*/true);
- for (CrossClusterCall &Call : RemoteClusterCalls)
+ for (CrossClusterCall &Call : LongThunkCalls)
relaxCall(Call, /*IsShort=*/false);
- if (!AdjacentClusterCalls.empty())
- BC.outs() << "BOLT-INFO: relaxed " << AdjacentClusterCalls.size()
- << " adjacent cluster calls with thunks\n";
+ if (!ShortThunkCalls.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << ShortThunkCalls.size()
+ << " short cluster calls with thunks\n";
- if (!RemoteClusterCalls.empty())
- BC.outs() << "BOLT-INFO: relaxed " << RemoteClusterCalls.size()
- << " remote cluster calls with thunks\n";
+ if (!LongThunkCalls.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << LongThunkCalls.size()
+ << " long cluster calls with thunks\n";
if (NumShortThunks)
BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 4c33e964683ad..8bbc1b7bc85d1 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -55,14 +55,16 @@ hot:
# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
## Check the number of thunks created in other modes.
-# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: 3 long thunks created
-# CHECK-BOLT: BOLT-INFO: 1 long thunks reused
+# CHECK-BOLT: BOLT-INFO: relaxed 2 short cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 2 long cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: 1 short thunks created
+# CHECK-BOLT: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT: BOLT-INFO: 1 short thunks reused
-# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 1 adjacent cluster calls with thunks
-# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 3 remote cluster calls with thunks
-# CHECK-BOLT-HOT-END: BOLT-INFO: 1 short thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 2 short cluster calls with thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 2 long cluster calls with thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: 2 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: 1 long thunks created
# CHECK-BOLT-HOT-END: BOLT-INFO: 1 long thunks reused
## Check that correct veneers are used depending on the target proximity.
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 7a646059df429..6af9d3b223861 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -1,22 +1,21 @@
## Relax direct calls using thunks across function fragment clusters. A/B/C/D
## contain 42MiB islands. With --split-functions, BOLT places the cold blocks
## of A/B/C/D in .text.cold. The test exercises direct adjacent-cluster calls,
-## forward and backward short thunks for adjacent clusters, and long thunk reuse
-## across remote clusters. With the default 124MiB function-fragment cluster
-## size, call relaxation sees:
+## forward and backward short thunks, and long thunk reuse across remote
+## clusters. With the default 124MiB function-fragment cluster size, call
+## relaxation sees:
##
-## A -> B same cluster, direct
-## B -> C adjacent cluster but close enough, direct
-## C -> A adjacent cluster but close enough, direct
-## A -> D forward short thunk
-## D -> A backward short thunk
+## A -> B direct same cluster
+## B -> C direct forward
+## C -> A direct backward
+## A -> D forward short
+## D -> A backward short
##
-## B.cold -> B normal: backward long thunk
-## C.cold -> B normal: shares backward long thunk to B
-## D.cold -> B normal: shares backward long thunk to B
-##
-## B.cold -> D HFE: forward long thunk
-## D.cold -> D HFE: shares forward long thunk to D
+## B.cold -> A normal: backward long shared; HFE: forward short
+## C.cold -> A normal: backward long shared; HFE: direct forward
+## D.cold -> A normal: backward long shared; HFE: direct forward
+## B.cold -> D normal: direct backward; HFE: forward long shared
+## D.cold -> D normal: backward long; HFE: forward long shared
##
## normal layout
## -------------
@@ -45,10 +44,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_backward_long_call_B,__AArch64_backward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_backward_long_call_A,__AArch64_backward_long_call_D \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_B,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_forward_long_call_B,__AArch64_forward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_A,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_forward_long_call_D \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
@@ -64,8 +63,8 @@
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
-# CHECK-BOLT: BOLT-INFO: relaxed 2 adjacent cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: relaxed 4 remote cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 2 short cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 4 long cluster calls with thunks
# CHECK-BOLT: BOLT-INFO: 2 short thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks reused
@@ -85,10 +84,10 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 adjacent cluster calls with thunks
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 remote cluster calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 short cluster calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 2 long cluster calls with thunks
# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
-# CHECK-BOLT-HFE: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
# CHECK-BOLT-HFE: BOLT-INFO: 16 branch thunks created
@@ -128,7 +127,7 @@ B:
ret
.B_cold:
mov x0, #2
- bl B
+ bl A
bl D
b .B_ret
.space 0x2a00000
@@ -147,7 +146,7 @@ C:
ret
.C_cold:
mov x0, #3
- bl B
+ bl A
b .C_ret
.space 0x2a00000
.size C, .-C
@@ -165,7 +164,7 @@ D:
ret
.D_cold:
mov x0, #4
- bl B
+ bl A
bl D
b .D_ret
.space 0x2a00000
@@ -197,10 +196,10 @@ D:
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <D>
-# CHECK-OUTPUT: <__AArch64_backward_long_call_B>:
+# CHECK-OUTPUT: <__AArch64_backward_long_call_A>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
@@ -212,42 +211,37 @@ D:
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_B>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_D>
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_A>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_A>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_B>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_B>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <B>
-
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D>
>From 8cacb99f51a700e8f1a1757f061ba484aa1e56c3 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sun, 6 Sep 2026 17:25:17 +0000
Subject: [PATCH 10/22] Shorten branch thunk chains when hops are in range
---
bolt/lib/Passes/LongJmp.cpp | 46 +++++++++--
.../AArch64/relax-branches-with-thunk-chain.s | 80 ++++++++-----------
.../test/AArch64/relax-cross-fragment-calls.s | 4 +-
3 files changed, 74 insertions(+), 56 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index cc53a3cff1708..7fde79e9692d3 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -15,6 +15,7 @@
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
#include <algorithm>
@@ -1453,6 +1454,8 @@ void LongJmpPass::relaxUnconditionalBranches(
struct CrossClusterBranch {
MCInst *Inst;
const MCSymbol *TargetSymbol;
+ uint64_t SourceOffset;
+ uint64_t TargetOffset;
unsigned SourceCluster;
unsigned TargetCluster;
};
@@ -1494,8 +1497,9 @@ void LongJmpPass::relaxUnconditionalBranches(
if (isWithinClusterRange(SourceOffset, TargetOffset))
continue;
- CrossClusterBranches.push_back(
- {&Inst, TargetSymbol, SourceCluster, TargetCluster});
+ CrossClusterBranches.push_back({&Inst, TargetSymbol, SourceOffset,
+ TargetOffset, SourceCluster,
+ TargetCluster});
}
}
}
@@ -1540,15 +1544,41 @@ void LongJmpPass::relaxUnconditionalBranches(
[&](const CrossClusterBranch &Branch) -> const MCSymbol * {
const unsigned SourceCluster = Branch.SourceCluster;
const unsigned TargetCluster = Branch.TargetCluster;
- BinaryFunction *FirstThunk = nullptr;
- const MCSymbol *NextTarget = Branch.TargetSymbol;
-
const bool IsForward = SourceCluster < TargetCluster;
const unsigned NumHops = IsForward ? TargetCluster - SourceCluster
: SourceCluster - TargetCluster;
- for (unsigned I = 0; I < NumHops; ++I) {
- const unsigned Cluster =
- IsForward ? TargetCluster - I - 1 : TargetCluster + I + 1;
+
+ auto getClusterAtHop = [&](const unsigned Hop) {
+ return IsForward ? SourceCluster + Hop : SourceCluster - Hop;
+ };
+
+ auto getThunkOffset = [&](const unsigned Cluster) {
+ const FragmentCluster &FC = Clusters[Cluster];
+ return IsForward ? FC.getEndOffset() : FC.StartOffset;
+ };
+
+ SmallVector<unsigned> ThunkClusters;
+ uint64_t CurrentOffset = Branch.SourceOffset;
+ unsigned NextHop = 0;
+ while (!isWithinClusterRange(CurrentOffset, Branch.TargetOffset)) {
+ unsigned BestHop = -1u;
+ for (unsigned Hop = NextHop; Hop < NumHops; ++Hop) {
+ const unsigned Cluster = getClusterAtHop(Hop);
+ if (!isWithinClusterRange(CurrentOffset, getThunkOffset(Cluster)))
+ break;
+ BestHop = Hop;
+ }
+
+ assert(BestHop != -1u && "expected reachable branch thunk");
+ const unsigned BestCluster = getClusterAtHop(BestHop);
+ ThunkClusters.push_back(BestCluster);
+ CurrentOffset = getThunkOffset(BestCluster);
+ NextHop = BestHop + 1;
+ }
+
+ BinaryFunction *FirstThunk = nullptr;
+ const MCSymbol *NextTarget = Branch.TargetSymbol;
+ for (const unsigned Cluster : llvm::reverse(ThunkClusters)) {
FirstThunk = getOrCreateBranchThunk(
Clusters[Cluster], Branch.TargetSymbol, NextTarget, IsForward);
NextTarget = FirstThunk->getSymbol();
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index 5626eb6e90bbc..b376ae570f149 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -30,10 +30,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_forward_branch_chain_8,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3,__AArch64_backward_branch_chain_6,__AArch64_backward_branch_chain_7,__AArch64_backward_branch_chain_9 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_2,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_backward_branch_chain_1,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5,__AArch64_backward_branch_chain_7 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_2,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_forward_branch_chain_7,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_1,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -47,7 +47,7 @@
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes
# CHECK-BOLT: BOLT-INFO: relaxed 7 cross-cluster branches
-# CHECK-BOLT: BOLT-INFO: 10 branch thunks created
+# CHECK-BOLT: BOLT-INFO: 8 branch thunks created
# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
@@ -61,7 +61,7 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 8 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 6 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
@@ -179,68 +179,62 @@ pad_hot_3:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
+# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[B_ALT:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[B_RET:[0-9a-f]+]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_5>
-# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64_forward_branch_chain_5>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
+# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64_forward_branch_chain_3>
# CHECK-OUTPUT-NEXT: [[B_RET]]: {{.*}} ret
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_1>:
-# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_0>:
+# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_5>:
-# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_4>
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_3>:
+# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_2>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_2>:
-# CHECK-OUTPUT-NEXT: [[A_BW1:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_1>:
+# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_6>:
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_4>:
# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x10>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[C_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_8>
+# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_0>:
-# CHECK-OUTPUT-NEXT: [[A_FW1]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
-
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_4>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_2>:
# CHECK-OUTPUT-NEXT: [[B_FW1]]: {{.*}} b 0x[[B_COLD:[0-9a-f]+]] <B.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_8>:
+# CHECK-OUTPUT: <__AArch64_forward_branch_chain_6>:
# CHECK-OUTPUT-NEXT: [[C_FW0]]: {{.*}} b 0x[[C_COLD:[0-9a-f]+]] <C.cold.0>
# CHECK-OUTPUT: Disassembly of section .text.cold:
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_3>:
-# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_BW1]] <__AArch64_backward_branch_chain_2>
+# CHECK-OUTPUT: <__AArch64_backward_branch_chain_5>:
+# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64_backward_branch_chain_4>
# CHECK-OUTPUT: <__AArch64_backward_branch_chain_7>:
-# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64_backward_branch_chain_6>
-
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_9>:
# CHECK-OUTPUT-NEXT: [[C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[C_RET]] <C+0x4>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT-NEXT: [[A_COLD]]: {{.*}} mov x0, #0x1
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_branch_chain_3>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_branch_chain_1>
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: [[B_COLD]]: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64_backward_branch_chain_7>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64_backward_branch_chain_5>
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: [[C_COLD]]: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64_backward_branch_chain_9>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64_backward_branch_chain_7>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: [[D_COLD]]: {{.*}} mov x0, #0x4
@@ -259,24 +253,24 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_7>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_5>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_2>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_1>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_7>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_4>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_4>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_2>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
# CHECK-HFE-OUTPUT: <A>:
@@ -291,24 +285,18 @@ pad_hot_3:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_2>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW1]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_6>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_4>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_1>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_0>
-
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_4>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_3>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_2>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW0]] <__AArch64_backward_branch_chain_1>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_0>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_5>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_3>
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 6af9d3b223861..253da8072d6b8 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -69,7 +69,7 @@
# CHECK-BOLT: BOLT-INFO: 2 long thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks reused
# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
-# CHECK-BOLT: BOLT-INFO: 16 branch thunks created
+# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
@@ -90,7 +90,7 @@
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks reused
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
-# CHECK-BOLT-HFE: BOLT-INFO: 16 branch thunks created
+# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
# CHECK-SECTIONS: .text
# CHECK-SECTIONS: .text.cold
>From 9aed40d7b6ec9a529ae97f6f1b8e43bdb8861b7a Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Mon, 7 Sep 2026 15:35:48 +0000
Subject: [PATCH 11/22] refactor branch collection and get rid of relax-plt
---
bolt/include/bolt/Passes/LongJmp.h | 41 ++++-
bolt/lib/Passes/LongJmp.cpp | 287 +++++++++++------------------
2 files changed, 143 insertions(+), 185 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 126785f0c2b83..ff2f56a0b3e75 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -130,25 +130,50 @@ class LongJmpPass : public BinaryFunctionPass {
};
struct FragmentClusterLayout {
+ struct Position {
+ unsigned Cluster;
+ uint64_t Offset;
+ };
+
SmallVector<FragmentCluster, 4> Clusters;
- DenseMap<const BinaryBasicBlock *, unsigned> BBToCluster;
- DenseMap<const BinaryBasicBlock *, uint64_t> BBToOffset;
- DenseMap<const MCSymbol *, unsigned> SymToCluster;
- DenseMap<const MCSymbol *, uint64_t> SymToOffset;
+ DenseMap<const BinaryBasicBlock *, Position> BBLayout;
+ DenseMap<const MCSymbol *, Position> SymLayout;
+ };
+
+ struct CrossClusterReference {
+ MCInst *Inst;
+ const MCSymbol *TargetSymbol;
+ uint64_t SourceOffset;
+ uint64_t TargetOffset;
+ unsigned SourceCluster;
+ unsigned TargetCluster;
+ };
+
+ struct ClusteredReferences {
+ SmallVector<CrossClusterReference> ShortThunkCalls;
+ SmallVector<CrossClusterReference> OutOfLayoutLongThunkCalls;
+ SmallVector<SmallVector<CrossClusterReference>, 4> LongThunkCallsByDistance;
+ SmallVector<CrossClusterReference> CrossClusterBranches;
};
FragmentClusterLayout
buildClusterLayout(BinaryContext &BC,
const BinaryFunctionListType &OutputFunctions);
+ /// Collect call and branch references that need cluster-level relaxation.
+ ClusteredReferences
+ collectClusteredReferences(BinaryContext &BC,
+ const BinaryFunctionListType &OutputFunctions,
+ const FragmentClusterLayout &Layout);
+
/// Relax calls using function fragment clusters.
- void relaxCalls(BinaryContext &BC, BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout);
+ void relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
+ ClusteredReferences &References);
/// Relax direct unconditional branches using function fragment clusters.
void relaxUnconditionalBranches(BinaryContext &BC,
- BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout);
+ FragmentClusterLayout &Layout,
+ ClusteredReferences &References);
/// Insert all thunks owned by the fragment cluster layout.
void insertClusterThunks(BinaryFunctionListType &OutputFunctions,
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 7fde79e9692d3..ab9beaadef77e 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -39,10 +39,6 @@ static cl::opt<bool>
cl::desc("run experimental relaxation pass"),
cl::init(false), cl::cat(BoltOptCategory));
-static cl::opt<bool> RelaxPLT("relax-plt",
- cl::desc("indicate PLT proximity to hot text"),
- cl::init(false), cl::cat(BoltOptCategory));
-
static cl::opt<unsigned long long> MaxClusterSize(
"max-cluster-size",
cl::desc("maximum estimated size of a function fragment cluster in bytes"),
@@ -81,6 +77,10 @@ static bool isWithinClusterRange(uint64_t SourceOffset, uint64_t TargetOffset) {
return Distance < opts::MaxClusterSize;
}
+static unsigned getClusterDistance(unsigned A, unsigned B) {
+ return A > B ? A - B : B - A;
+}
+
static BinaryBasicBlock *getBBAtHotColdSplitPoint(BinaryFunction &Func) {
if (!Func.isSplit() || Func.empty())
return nullptr;
@@ -1149,25 +1149,18 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
// Map primary entry points.
if (FF.isMainFragment())
- for (const MCSymbol *Symbol : BF.getSymbols()) {
- Layout.SymToCluster[Symbol] = ClusterNum;
- Layout.SymToOffset[Symbol] = FragmentOffset;
- }
+ for (const MCSymbol *Symbol : BF.getSymbols())
+ Layout.SymLayout[Symbol] = {ClusterNum, FragmentOffset};
uint64_t BBOffset = FragmentOffset;
for (const BinaryBasicBlock *BB : FF) {
- Layout.BBToCluster[BB] = ClusterNum;
- Layout.BBToOffset[BB] = BBOffset;
- if (const MCSymbol *Label = BB->getLabel()) {
- Layout.SymToCluster[Label] = ClusterNum;
- Layout.SymToOffset[Label] = BBOffset;
- }
+ Layout.BBLayout[BB] = {ClusterNum, BBOffset};
+ if (const MCSymbol *Label = BB->getLabel())
+ Layout.SymLayout[Label] = {ClusterNum, BBOffset};
// Map secondary entry points.
- if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB)) {
- Layout.SymToCluster[EntrySymbol] = ClusterNum;
- Layout.SymToOffset[EntrySymbol] = BBOffset;
- }
+ if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
+ Layout.SymLayout[EntrySymbol] = {ClusterNum, BBOffset};
BBOffset += BB->estimateSize();
}
@@ -1198,51 +1191,17 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
<< "BOLT-INFO: " << FC.Size << " estimated bytes\n";
}
- if (opts::RelaxPLT) {
- // Populate one of the clusters with PLT functions based on the proximity of
- // the PLT section to avoid unneeded thunk redirection.
- const unsigned PLTCluster =
- opts::UseOldText ? Layout.Clusters.size() - 1 : 0;
- for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions()))
- if (BF.isPLTFunction()) {
- for (const MCSymbol *Symbol : BF.getSymbols())
- Layout.SymToCluster[Symbol] = PLTCluster;
- BF.forEachEntryPoint(
- [&](uint64_t, const MCSymbol *EntrySymbol) -> bool {
- Layout.SymToCluster[EntrySymbol] = PLTCluster;
- return true;
- });
- }
- }
-
return Layout;
}
-void LongJmpPass::relaxCalls(BinaryContext &BC,
- BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout) {
+LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
+ BinaryContext &BC, const BinaryFunctionListType &OutputFunctions,
+ const FragmentClusterLayout &Layout) {
auto &Clusters = Layout.Clusters;
- auto &BBToCluster = Layout.BBToCluster;
- auto &BBToOffset = Layout.BBToOffset;
- auto &SymToCluster = Layout.SymToCluster;
- auto &SymToOffset = Layout.SymToOffset;
-
- struct CrossClusterCall {
- MCInst *Inst;
- const MCSymbol *TargetSymbol;
- uint64_t SourceOffset;
- uint64_t TargetOffset;
- unsigned SourceCluster;
- unsigned TargetCluster;
- };
-
- auto clusterDistance = [](const CrossClusterCall &Call) {
- return Call.SourceCluster > Call.TargetCluster
- ? Call.SourceCluster - Call.TargetCluster
- : Call.TargetCluster - Call.SourceCluster;
- };
+ auto &BBLayout = Layout.BBLayout;
+ auto &SymLayout = Layout.SymLayout;
- auto canUseShortThunk = [&](const CrossClusterCall &Call) {
+ auto canUseShortThunk = [&](const CrossClusterReference &Call) {
if (Call.TargetCluster == -1u)
return false;
@@ -1254,62 +1213,96 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
isWithinClusterRange(ThunkOffset, Call.TargetOffset);
};
- SmallVector<CrossClusterCall> ShortThunkCalls;
- SmallVector<CrossClusterCall> LongThunkCalls;
+ ClusteredReferences References;
+ References.LongThunkCallsByDistance.resize(Clusters.size());
+ DenseMap<const MCSymbol *, bool> ResolvableTargetCache;
+ auto isResolvableTarget = [&](const MCSymbol *TargetSymbol) {
+ auto It = ResolvableTargetCache.find(TargetSymbol);
+ if (It != ResolvableTargetCache.end())
+ return It->second;
+
+ const bool Resolvable = BC.getFunctionForSymbol(TargetSymbol) ||
+ BC.getSymbolValue(*TargetSymbol);
+ ResolvableTargetCache[TargetSymbol] = Resolvable;
+ return Resolvable;
+ };
+
for (BinaryFunction *BF : OutputFunctions) {
if (!BC.shouldEmit(*BF) || BF->isPatch())
continue;
for (BinaryBasicBlock &BB : *BF) {
- auto SourceIt = BBToCluster.find(&BB);
- if (SourceIt == BBToCluster.end())
+ auto SourceIt = BBLayout.find(&BB);
+ if (SourceIt == BBLayout.end())
continue;
- const unsigned SourceCluster = SourceIt->second;
- // The cluster lookup above guarantees this BB has a layout offset.
- uint64_t InstOffset = BBToOffset[&BB];
+ const FragmentClusterLayout::Position Source = SourceIt->second;
+ uint64_t InstOffset = Source.Offset;
for (MCInst &Inst : BB) {
const uint64_t SourceOffset = InstOffset;
- InstOffset += BC.computeInstructionSize(Inst);
+ if (!BC.MIB->isPseudo(Inst))
+ InstOffset += 4;
- if (!BC.MIB->isCall(Inst) && !BC.MIB->isUnconditionalBranch(Inst))
+ const bool IsCall = BC.MIB->isCall(Inst);
+ const bool IsUncondBranch = BC.MIB->isUnconditionalBranch(Inst);
+ if (!IsCall && !IsUncondBranch)
continue;
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
if (!TargetSymbol)
continue;
- auto It = SymToCluster.find(TargetSymbol);
- const bool Found = It != SymToCluster.end();
- // Known plain branches use B-only thunk chains.
- // Unknown address-only branches may use tail-call thunks.
- if (BC.MIB->isUnconditionalBranch(Inst) && Found)
+ auto TargetIt = SymLayout.find(TargetSymbol);
+ const bool Found = TargetIt != SymLayout.end();
+ // Unmapped targets use maximum offset and are treated as out of range.
+ const FragmentClusterLayout::Position Target =
+ Found ? TargetIt->second
+ : FragmentClusterLayout::Position{-1u, -1ULL};
+ const CrossClusterReference Reference{&Inst, TargetSymbol,
+ SourceOffset, Target.Offset,
+ Source.Cluster, Target.Cluster};
+ if (IsUncondBranch && Found) {
+ if (Source.Cluster == Target.Cluster)
+ continue;
+ if (isWithinClusterRange(SourceOffset, Target.Offset))
+ continue;
+
+ References.CrossClusterBranches.push_back(Reference);
continue;
+ }
- if (!BC.getFunctionForSymbol(TargetSymbol) &&
- !BC.getSymbolValue(*TargetSymbol))
+ if (!Found && !isResolvableTarget(TargetSymbol))
continue;
- // If not found use -1 so the computed distance is out of range.
- unsigned TargetCluster = Found ? It->second : -1u;
- if (TargetCluster == SourceCluster)
+ if (Target.Cluster == Source.Cluster)
continue;
- // A target with a cluster was mapped by the layout builder, so it
- // also has an offset. Otherwise use -1 so the computed distance is
- // conservatively treated as out of range.
- const uint64_t TargetOffset = Found ? SymToOffset[TargetSymbol] : -1ULL;
- if (isWithinClusterRange(SourceOffset, TargetOffset))
+ if (isWithinClusterRange(SourceOffset, Target.Offset))
continue;
- CrossClusterCall Call{&Inst, TargetSymbol, SourceOffset,
- TargetOffset, SourceCluster, TargetCluster};
- auto &Calls = canUseShortThunk(Call) ? ShortThunkCalls : LongThunkCalls;
- Calls.push_back(Call);
+ if (canUseShortThunk(Reference)) {
+ References.ShortThunkCalls.push_back(Reference);
+ } else if (Reference.TargetCluster == -1u) {
+ References.OutOfLayoutLongThunkCalls.push_back(Reference);
+ } else {
+ const unsigned Distance = getClusterDistance(Reference.SourceCluster,
+ Reference.TargetCluster);
+ References.LongThunkCallsByDistance[Distance].push_back(Reference);
+ }
}
}
}
+ return References;
+}
+
+void LongJmpPass::relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
+ ClusteredReferences &References) {
+ auto &Clusters = Layout.Clusters;
+ auto &ShortThunkCalls = References.ShortThunkCalls;
+ auto &OutOfLayoutLongThunkCalls = References.OutOfLayoutLongThunkCalls;
+ auto &LongThunkCallsByDistance = References.LongThunkCallsByDistance;
+
size_t NumShortThunks = 0;
size_t NumLongThunks = 0;
size_t NumShortThunksReused = 0;
@@ -1364,19 +1357,14 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
// Reuse long thunks hosted at the adjacent cluster boundary.
if (!IsShort) {
FragmentCluster *ReuseCluster = nullptr;
- BinaryFunctionListType *ReuseThunkList = nullptr;
- if (IsForward && SourceCluster > 0) {
+ if (IsForward && SourceCluster > 0)
ReuseCluster = &Clusters[SourceCluster - 1];
- ReuseThunkList = &ReuseCluster->ForwardThunkList;
- } else if (!IsForward && SourceCluster + 1 < Clusters.size()) {
+ else if (!IsForward && SourceCluster + 1 < Clusters.size())
ReuseCluster = &Clusters[SourceCluster + 1];
- ReuseThunkList = &ReuseCluster->BackwardThunkList;
- }
if (ReuseCluster) {
auto It = ReuseCluster->CallThunks.find(TargetSymbol);
- if (It != ReuseCluster->CallThunks.end() &&
- llvm::is_contained(*ReuseThunkList, It->second)) {
+ if (It != ReuseCluster->CallThunks.end()) {
BinaryFunction *Thunk = It->second;
++NumLongThunksReused;
registerCallThunk(FC, TargetSymbol, Thunk);
@@ -1387,11 +1375,9 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort, IsForward);
- FragmentCluster &ThunkCluster = Clusters[SourceCluster];
- Thunk->setCodeSectionName(ThunkCluster.getThunkSectionName(IsForward));
- BinaryFunctionListType &ThunkList = IsForward
- ? ThunkCluster.ForwardThunkList
- : ThunkCluster.BackwardThunkList;
+ Thunk->setCodeSectionName(FC.getThunkSectionName(IsForward));
+ BinaryFunctionListType &ThunkList =
+ IsForward ? FC.ForwardThunkList : FC.BackwardThunkList;
ThunkList.push_back(Thunk);
// Register thunks for all symbols associated with the function.
@@ -1399,32 +1385,34 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
return Thunk;
};
- // Process farthest calls first so closer remote calls can reuse long thunks
- // already placed at adjacent cluster boundaries.
- llvm::stable_sort(LongThunkCalls,
- [&](const CrossClusterCall &A, const CrossClusterCall &B) {
- return clusterDistance(A) > clusterDistance(B);
- });
-
- auto relaxCall = [&](CrossClusterCall &Call, bool IsShort) {
+ auto relaxCall = [&](CrossClusterReference &Call, bool IsShort) {
const bool IsForward = Call.SourceCluster < Call.TargetCluster;
BinaryFunction *Thunk = getOrCreateCallThunk(
Call.TargetSymbol, Call.SourceCluster, IsShort, IsForward);
BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
};
- for (CrossClusterCall &Call : ShortThunkCalls)
+ for (CrossClusterReference &Call : ShortThunkCalls)
relaxCall(Call, /*IsShort=*/true);
- for (CrossClusterCall &Call : LongThunkCalls)
+ // Process out-of-layout targets first, then known targets from farthest to
+ // nearest, so closer calls can reuse long thunks at cluster boundaries.
+ for (CrossClusterReference &Call : OutOfLayoutLongThunkCalls)
relaxCall(Call, /*IsShort=*/false);
+ size_t NumLongThunkCalls = OutOfLayoutLongThunkCalls.size();
+ for (auto &Calls : llvm::reverse(LongThunkCallsByDistance)) {
+ NumLongThunkCalls += Calls.size();
+ for (CrossClusterReference &Call : Calls)
+ relaxCall(Call, /*IsShort=*/false);
+ }
+
if (!ShortThunkCalls.empty())
BC.outs() << "BOLT-INFO: relaxed " << ShortThunkCalls.size()
<< " short cluster calls with thunks\n";
- if (!LongThunkCalls.empty())
- BC.outs() << "BOLT-INFO: relaxed " << LongThunkCalls.size()
+ if (NumLongThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << NumLongThunkCalls
<< " long cluster calls with thunks\n";
if (NumShortThunks)
@@ -1442,67 +1430,11 @@ void LongJmpPass::relaxCalls(BinaryContext &BC,
<< " long thunks reused\n";
}
-void LongJmpPass::relaxUnconditionalBranches(
- BinaryContext &BC, BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout) {
+void LongJmpPass::relaxUnconditionalBranches(BinaryContext &BC,
+ FragmentClusterLayout &Layout,
+ ClusteredReferences &References) {
auto &Clusters = Layout.Clusters;
- auto &BBToCluster = Layout.BBToCluster;
- auto &BBToOffset = Layout.BBToOffset;
- auto &SymToCluster = Layout.SymToCluster;
- auto &SymToOffset = Layout.SymToOffset;
-
- struct CrossClusterBranch {
- MCInst *Inst;
- const MCSymbol *TargetSymbol;
- uint64_t SourceOffset;
- uint64_t TargetOffset;
- unsigned SourceCluster;
- unsigned TargetCluster;
- };
-
- SmallVector<CrossClusterBranch> CrossClusterBranches;
- for (BinaryFunction *BF : OutputFunctions) {
- if (!BC.shouldEmit(*BF) || BF->isPatch())
- continue;
-
- for (BinaryBasicBlock &BB : *BF) {
- auto SourceIt = BBToCluster.find(&BB);
- if (SourceIt == BBToCluster.end())
- continue;
- const unsigned SourceCluster = SourceIt->second;
- // The cluster lookup above guarantees this BB has a layout offset.
- uint64_t InstOffset = BBToOffset[&BB];
-
- for (MCInst &Inst : BB) {
- const uint64_t SourceOffset = InstOffset;
- InstOffset += BC.computeInstructionSize(Inst);
-
- if (!BC.MIB->isUnconditionalBranch(Inst))
- continue;
-
- const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
- if (!TargetSymbol)
- continue;
-
- auto TargetIt = SymToCluster.find(TargetSymbol);
- if (TargetIt == SymToCluster.end())
- continue;
-
- const unsigned TargetCluster = TargetIt->second;
- if (SourceCluster == TargetCluster)
- continue;
-
- // The cluster lookup above guarantees this symbol has a layout offset.
- const uint64_t TargetOffset = SymToOffset[TargetSymbol];
- if (isWithinClusterRange(SourceOffset, TargetOffset))
- continue;
-
- CrossClusterBranches.push_back({&Inst, TargetSymbol, SourceOffset,
- TargetOffset, SourceCluster,
- TargetCluster});
- }
- }
- }
+ auto &CrossClusterBranches = References.CrossClusterBranches;
size_t NumBranchThunks = 0;
size_t NumBranchThunksReused = 0;
@@ -1541,7 +1473,7 @@ void LongJmpPass::relaxUnconditionalBranches(
};
auto getOrCreateBranchThunkChain =
- [&](const CrossClusterBranch &Branch) -> const MCSymbol * {
+ [&](const CrossClusterReference &Branch) -> const MCSymbol * {
const unsigned SourceCluster = Branch.SourceCluster;
const unsigned TargetCluster = Branch.TargetCluster;
const bool IsForward = SourceCluster < TargetCluster;
@@ -1588,7 +1520,7 @@ void LongJmpPass::relaxUnconditionalBranches(
return FirstThunk->getSymbol();
};
- for (const CrossClusterBranch &Branch : CrossClusterBranches) {
+ for (const CrossClusterReference &Branch : CrossClusterBranches) {
const MCSymbol *Target = getOrCreateBranchThunkChain(Branch);
BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
}
@@ -1634,11 +1566,10 @@ void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
return !A.IsForward && B.IsForward;
});
- for (ThunkInsertion &Insertion : Insertions) {
+ for (ThunkInsertion &Insertion : Insertions)
OutputFunctions.insert(
std::next(OutputFunctions.begin(), Insertion.Position),
Insertion.ThunkList->begin(), Insertion.ThunkList->end());
- }
}
void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
@@ -1648,8 +1579,10 @@ void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
if (Layout.Clusters.empty())
return;
- relaxCalls(BC, OutputFunctions, Layout);
- relaxUnconditionalBranches(BC, OutputFunctions, Layout);
+ ClusteredReferences References =
+ collectClusteredReferences(BC, OutputFunctions, Layout);
+ relaxCalls(BC, Layout, References);
+ relaxUnconditionalBranches(BC, Layout, References);
insertClusterThunks(OutputFunctions, Layout);
LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
>From 0629e634608f8a9dd6244d0b7e6ce2c0b92a8240 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 8 Sep 2026 09:10:11 +0000
Subject: [PATCH 12/22] do not sort thousands of fragments, only their sections
---
bolt/lib/Passes/LongJmp.cpp | 48 +++++++++++++++++++++++++------------
1 file changed, 33 insertions(+), 15 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index ab9beaadef77e..bb53453195c80 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -17,6 +17,7 @@
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/Support/MathExtras.h"
#include <algorithm>
@@ -1070,7 +1071,16 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
};
FragmentClusterLayout Layout;
- SmallVector<OutputFragment> OrderedFragments;
+ SmallVector<SmallVector<OutputFragment>, 4> FragmentsBySection;
+ StringMap<size_t> SectionToBucket;
+ auto addOrderedFragment = [&](OutputFragment &&Fragment) {
+ auto [It, Inserted] = SectionToBucket.try_emplace(
+ Fragment.SectionName, FragmentsBySection.size());
+ if (Inserted)
+ FragmentsBySection.emplace_back();
+ FragmentsBySection[It->second].push_back(std::move(Fragment));
+ };
+
for (size_t I = 0; I < OutputFunctions.size(); ++I) {
BinaryFunction *BF = OutputFunctions[I];
if (!BC.shouldEmit(*BF) || BF->isPatch())
@@ -1080,18 +1090,26 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
if (FF.empty() && !BF->hasConstantIsland())
continue;
- OrderedFragments.push_back({&FF, I,
- BF->getCodeSectionName(FF.getFragmentNum()),
- estimateFragmentSize(*BF, FF)});
+ addOrderedFragment({&FF, I, BF->getCodeSectionName(FF.getFragmentNum()),
+ estimateFragmentSize(*BF, FF)});
}
}
// Model final output layout by grouping function fragments in output section
// order. Within each section, fragments remain in OutputFunctions order.
- llvm::stable_sort(
- OrderedFragments, [&](const OutputFragment &A, const OutputFragment &B) {
- return BC.compareSectionNames(A.SectionName, B.SectionName);
- });
+ SmallVector<size_t, 4> SectionOrder;
+ for (size_t I = 0; I < FragmentsBySection.size(); ++I)
+ SectionOrder.push_back(I);
+
+ llvm::sort(SectionOrder, [&](size_t A, size_t B) {
+ return BC.compareSectionNames(FragmentsBySection[A].front().SectionName,
+ FragmentsBySection[B].front().SectionName);
+ });
+
+ SmallVector<const OutputFragment *> OrderedFragments;
+ for (size_t SectionIndex : SectionOrder)
+ for (const OutputFragment &Fragment : FragmentsBySection[SectionIndex])
+ OrderedFragments.push_back(&Fragment);
auto buildClusterRanges = [&]() {
SmallVector<FragmentRange> ClusterRanges;
@@ -1100,29 +1118,29 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
if (!opts::HotFunctionsAtEnd) {
size_t Begin = 0;
- uint64_t Size = OrderedFragments[0].Size;
+ uint64_t Size = OrderedFragments[0]->Size;
for (size_t I = 1; I < OrderedFragments.size(); ++I) {
- if (Size + OrderedFragments[I].Size > opts::MaxClusterSize) {
+ if (Size + OrderedFragments[I]->Size > opts::MaxClusterSize) {
ClusterRanges.push_back({Begin, I});
Begin = I;
Size = 0;
}
- Size += OrderedFragments[I].Size;
+ Size += OrderedFragments[I]->Size;
}
ClusterRanges.push_back({Begin, OrderedFragments.size()});
return ClusterRanges;
}
size_t End = OrderedFragments.size();
- uint64_t Size = OrderedFragments.back().Size;
+ uint64_t Size = OrderedFragments.back()->Size;
for (size_t I = End - 1; I > 0;) {
--I;
- if (Size + OrderedFragments[I].Size > opts::MaxClusterSize) {
+ if (Size + OrderedFragments[I]->Size > opts::MaxClusterSize) {
ClusterRanges.push_back({I + 1, End});
End = I + 1;
Size = 0;
}
- Size += OrderedFragments[I].Size;
+ Size += OrderedFragments[I]->Size;
}
ClusterRanges.push_back({0, End});
std::reverse(ClusterRanges.begin(), ClusterRanges.end());
@@ -1172,7 +1190,7 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
for (const FragmentRange &Range : buildClusterRanges()) {
Layout.Clusters.emplace_back();
for (size_t I = Range.Begin; I < Range.End; ++I)
- addFragmentToCluster(OrderedFragments[I]);
+ addFragmentToCluster(*OrderedFragments[I]);
}
if (Layout.Clusters.empty())
>From c4d11601e308f5de124606cad830d2e601ab44da Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Wed, 9 Sep 2026 00:30:57 +0000
Subject: [PATCH 13/22] do not treat unconditional branches to secondary
entries as tailcalls
---
bolt/include/bolt/Core/BinaryFunction.h | 6 +
bolt/lib/Passes/LongJmp.cpp | 37 ++---
.../tail-classified-body-branch-live-x16.S | 136 ++++++++++++++++++
3 files changed, 162 insertions(+), 17 deletions(-)
create mode 100644 bolt/test/AArch64/tail-classified-body-branch-live-x16.S
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 3865e87dc25ef..cf5f60bba3920 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -908,6 +908,12 @@ class BinaryFunction {
/// Returns whether there are any labels at Offset.
bool hasLabelAt(unsigned Offset) const { return Labels.count(Offset) != 0; }
+ /// Return the label at \p Offset, or nullptr if none exists.
+ MCSymbol *getLabelAtOffset(unsigned Offset) const {
+ auto It = Labels.find(Offset);
+ return It == Labels.end() ? nullptr : It->second;
+ }
+
/// Iterate over all jump tables associated with this function.
iterator_range<std::map<uint64_t, JumpTable *>::const_iterator>
jumpTables() const {
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index bb53453195c80..1f51280256b25 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1173,12 +1173,13 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
uint64_t BBOffset = FragmentOffset;
for (const BinaryBasicBlock *BB : FF) {
Layout.BBLayout[BB] = {ClusterNum, BBOffset};
+ // Map the local BB label.
if (const MCSymbol *Label = BB->getLabel())
Layout.SymLayout[Label] = {ClusterNum, BBOffset};
-
- // Map secondary entry points.
- if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))
- Layout.SymLayout[EntrySymbol] = {ClusterNum, BBOffset};
+ // Map the secondary entry point, which can differ from the BB label.
+ if (MCSymbol *Label = BF.getLabelAtOffset(BB->getOffset()))
+ if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(Label))
+ Layout.SymLayout[EntrySymbol] = {ClusterNum, BBOffset};
BBOffset += BB->estimateSize();
}
@@ -1233,16 +1234,10 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
ClusteredReferences References;
References.LongThunkCallsByDistance.resize(Clusters.size());
- DenseMap<const MCSymbol *, bool> ResolvableTargetCache;
- auto isResolvableTarget = [&](const MCSymbol *TargetSymbol) {
- auto It = ResolvableTargetCache.find(TargetSymbol);
- if (It != ResolvableTargetCache.end())
- return It->second;
-
- const bool Resolvable = BC.getFunctionForSymbol(TargetSymbol) ||
- BC.getSymbolValue(*TargetSymbol);
- ResolvableTargetCache[TargetSymbol] = Resolvable;
- return Resolvable;
+ auto isPrimaryEntryTarget = [&](const MCSymbol *TargetSymbol) {
+ uint64_t EntryID = 0;
+ const BinaryFunction *BF = BC.getFunctionForSymbol(TargetSymbol, &EntryID);
+ return BF && EntryID == 0;
};
for (BinaryFunction *BF : OutputFunctions) {
@@ -1262,6 +1257,7 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
InstOffset += 4;
const bool IsCall = BC.MIB->isCall(Inst);
+ const bool IsTailCall = BC.MIB->isTailCall(Inst);
const bool IsUncondBranch = BC.MIB->isUnconditionalBranch(Inst);
if (!IsCall && !IsUncondBranch)
continue;
@@ -1279,7 +1275,15 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
const CrossClusterReference Reference{&Inst, TargetSymbol,
SourceOffset, Target.Offset,
Source.Cluster, Target.Cluster};
- if (IsUncondBranch && Found) {
+ const bool UseBranchChain =
+ Found && (IsUncondBranch ||
+ (IsTailCall && !isPrimaryEntryTarget(TargetSymbol)));
+ if (UseBranchChain) {
+ // A direct B to a body entry may be annotated as a tail call. It is
+ // not an ABI call boundary, so use branch chains instead of call
+ // thunks that may clobber x16/x17.
+ if (IsTailCall)
+ BC.MIB->convertTailCallToJmp(Inst);
if (Source.Cluster == Target.Cluster)
continue;
if (isWithinClusterRange(SourceOffset, Target.Offset))
@@ -1289,8 +1293,7 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
continue;
}
- if (!Found && !isResolvableTarget(TargetSymbol))
- continue;
+ assert(IsCall && "expected call after branch-chain handling");
if (Target.Cluster == Source.Cluster)
continue;
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
new file mode 100644
index 0000000000000..9d2124fbfa40d
--- /dev/null
+++ b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
@@ -0,0 +1,136 @@
+## Check that a direct B to a function-body symbol is not relaxed as an
+## ABI tail call. Such a branch can carry live caller-saved registers, so
+## relaxation must use B-only branch chains instead of ADRP/BR call thunks.
+##
+## In the TAILCALL run, bar_body is at the function entry and the branch is an
+## ABI tail call. In the BRANCH run, bar_body is inside the function body and
+## must be relaxed as a branch.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t.tailcall -nostdlib
+# RUN: link_fdata --no-lbr %s %t.tailcall %t.tailcall.fdata
+# RUN: llvm-bolt %t.tailcall -o %t.tailcall.bolt \
+# RUN: --data %t.tailcall.fdata --relocs \
+# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
+# RUN: --skip-funcs='^bar$' \
+# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_long_call_bar \
+# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
+
+# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch \
+# RUN: -nostdlib
+# RUN: link_fdata --no-lbr %s %t.branch %t.branch.fdata
+# RUN: llvm-bolt %t.branch -o %t.branch.bolt \
+# RUN: --data %t.branch.fdata --relocs \
+# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
+# RUN: | FileCheck %s --check-prefix=BRANCH-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=foo,bar,bar_body,.bar_ret,padding,__AArch64_forward_branch_chain_0 \
+# RUN: %t.branch.bolt | FileCheck %s --check-prefix=BRANCH
+
+# TAILCALL-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
+# TAILCALL-BOLT: BOLT-INFO: starting experimental relaxation pass
+# TAILCALL-BOLT: BOLT-INFO: relaxed 1 long cluster calls with thunks
+# TAILCALL-BOLT: BOLT-INFO: 1 long thunks created
+
+# BRANCH-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
+# BRANCH-BOLT: BOLT-INFO: starting experimental relaxation pass
+# BRANCH-BOLT: BOLT-INFO: relaxed 1 cross-cluster branches
+# BRANCH-BOLT: BOLT-INFO: 1 branch thunks created
+
+ .text
+ .globl bar
+ .type bar, %function
+bar:
+#ifdef BRANCH
+.bar_entry:
+ b .bar_ret
+#endif
+ .globl bar_body
+ .type bar_body, %notype
+bar_body:
+ cmp x16, #42
+ cset w0, ne
+ ret
+#ifdef BRANCH
+.bar_ret:
+ ret
+#endif
+ .size bar, .-bar
+
+ .globl foo
+ .type foo, %function
+foo:
+.foo_entry:
+# FDATA: 1 foo #.foo_entry# 100
+ mov x16, #42
+ b bar_body
+ .space 0x4400000
+ .size foo, .-foo
+
+ .globl padding
+ .type padding, %function
+padding:
+.padding_entry:
+# FDATA: 1 padding #.padding_entry# 100
+ ret
+ .space 0x4400000
+ .size padding, .-padding
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# TAILCALL: Disassembly of section .bolt.org.text:
+#
+# TAILCALL: <bar>:
+# TAILCALL-NEXT: <bar_body>:
+# TAILCALL-NEXT: {{.*}} cmp x16, #0x2a
+# TAILCALL-NEXT: {{.*}} cset w0, ne
+# TAILCALL-NEXT: {{.*}} ret
+#
+# TAILCALL: Disassembly of section .text:
+#
+# TAILCALL: <foo>:
+# TAILCALL-NEXT: {{.*}} mov x16, #0x2a
+# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_long_call_bar>
+#
+# TAILCALL: <__AArch64_forward_long_call_bar>:
+# TAILCALL-NEXT: {{.*}} adrp x16,
+# TAILCALL-NEXT: {{.*}} add x16, x16,
+# TAILCALL-NEXT: {{.*}} br x16
+#
+# TAILCALL: <padding>:
+# TAILCALL-NEXT: {{.*}} ret
+
+
+# BRANCH: Disassembly of section .bolt.org.text:
+#
+# BRANCH: <bar_body>:
+# BRANCH-NEXT: {{.*}} cmp x16, #0x2a
+# BRANCH-NEXT: {{.*}} cset w0, ne
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: Disassembly of section .text:
+#
+# BRANCH: <foo>:
+# BRANCH-NEXT: {{.*}} mov x16, #0x2a
+# BRANCH-NEXT: {{.*}} b 0x[[CHAIN:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
+#
+# BRANCH: <__AArch64_forward_branch_chain_0>:
+# BRANCH-NEXT: [[CHAIN]]: {{.*}} b 0x[[BODY:[0-9a-f]+]] <bar+0x4>
+#
+# BRANCH: <padding>:
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: Disassembly of section .text.cold:
+#
+# BRANCH: <bar>:
+# BRANCH-NEXT: {{.*}} b 0x[[RET:[0-9a-f]+]] <.bar_ret>
+# BRANCH-NEXT: [[BODY]]: {{.*}} cmp x16, #0x2a
+# BRANCH-NEXT: {{.*}} cset w0, ne
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: <.bar_ret>:
+# BRANCH-NEXT: [[RET]]: {{.*}} ret
>From d9229baef28fd407d7b184c4c4f4765105efc882 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 10 Sep 2026 09:01:22 +0000
Subject: [PATCH 14/22] do not register long call thunks in clusters which only
reuse them
---
bolt/include/bolt/Passes/LongJmp.h | 2 +-
bolt/lib/Passes/LongJmp.cpp | 7 +-
.../AArch64/relax-call-long-thunk-reuse.s | 109 ++++++++++++++++++
bolt/test/AArch64/relax-calls.s | 4 +-
.../test/AArch64/relax-cross-fragment-calls.s | 42 +++----
.../tail-classified-body-branch-live-x16.S | 6 +-
6 files changed, 139 insertions(+), 31 deletions(-)
create mode 100644 bolt/test/AArch64/relax-call-long-thunk-reuse.s
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index ff2f56a0b3e75..c119f511c0683 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -113,7 +113,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Thunks located before this cluster.
BinaryFunctionListType BackwardThunkList;
- /// Call thunks used by this cluster.
+ /// Call thunks emitted by this cluster.
///
/// <Function Symbol> -> <Thunk Function>.
DenseMap<const MCSymbol *, BinaryFunction *> CallThunks;
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 1f51280256b25..95c44037904c1 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1331,18 +1331,18 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort,
bool IsForward) {
BinaryFunction *Thunk = nullptr;
+ const size_t ThunkNumber = IsShort ? NumShortThunks++ : NumLongThunks++;
std::string ThunkName =
(Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
- (IsShort ? "short_call_" : "long_call_") + TargetSymbol->getName())
+ (IsShort ? "short_call_" : "long_call_") + TargetSymbol->getName() +
+ "_" + Twine(ThunkNumber))
.str();
if (IsShort) {
- ++NumShortThunks;
Thunk = BC.createThunkBinaryFunction(ThunkName);
MCInst Inst;
BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
Thunk->addBasicBlock()->addInstruction(Inst);
} else {
- ++NumLongThunks;
Thunk = BC.createThunkBinaryFunction(ThunkName);
InstructionListType Instructions;
BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
@@ -1388,7 +1388,6 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
if (It != ReuseCluster->CallThunks.end()) {
BinaryFunction *Thunk = It->second;
++NumLongThunksReused;
- registerCallThunk(FC, TargetSymbol, Thunk);
return Thunk;
}
}
diff --git a/bolt/test/AArch64/relax-call-long-thunk-reuse.s b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
new file mode 100644
index 0000000000000..1942b55c38202
--- /dev/null
+++ b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
@@ -0,0 +1,109 @@
+## Check that long call thunk reuse only uses thunks emitted by the adjacent
+## cluster. Borrowed thunks must not be cached as if they were emitted by the
+## borrowing cluster, otherwise reuse can become transitive and pick a thunk at
+## the wrong cluster boundary.
+##
+## With --max-cluster-size=64, each function forms its own cluster:
+##
+## cluster 0: A
+## cluster 1: B
+## cluster 2: C -> A
+## cluster 3: D -> A
+## cluster 4: E -> A
+##
+## E creates a backward long thunk to A in cluster 4. D can reuse it from the
+## adjacent cluster boundary, but C needs a separate thunk in cluster 2.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-strip --strip-unneeded %t
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata \
+# RUN: --compact-code-model --relax-exp --max-cluster-size=64 \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,E,__AArch64_backward_long_call_A_0,__AArch64_backward_long_call_A_1 \
+# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+
+# CHECK-BOLT: BOLT-INFO: built 5 function fragment cluster(s)
+# CHECK-BOLT: BOLT-INFO: relaxed 3 long cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT: BOLT-INFO: 1 long thunks reused
+
+ .text
+ .globl A
+ .type A, %function
+A:
+.A_entry:
+# FDATA: 1 A #.A_entry# 100
+ ret
+ .space 0x30
+ .size A, .-A
+
+ .globl B
+ .type B, %function
+B:
+.B_entry:
+# FDATA: 1 B #.B_entry# 100
+ ret
+ .space 0x30
+ .size B, .-B
+
+ .globl C
+ .type C, %function
+C:
+.C_entry:
+# FDATA: 1 C #.C_entry# 100
+ bl A
+ ret
+ .space 0x30
+ .size C, .-C
+
+ .globl D
+ .type D, %function
+D:
+.D_entry:
+# FDATA: 1 D #.D_entry# 100
+ bl A
+ ret
+ .space 0x30
+ .size D, .-D
+
+ .globl E
+ .type E, %function
+E:
+.E_entry:
+# FDATA: 1 E #.E_entry# 100
+ bl A
+ ret
+ .space 0x30
+ .size E, .-E
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# CHECK-OUTPUT: <A>:
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <B>:
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <__AArch64_backward_long_call_A_1>:
+# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-OUTPUT: <C>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_1>
+
+# CHECK-OUTPUT: <D>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+
+# CHECK-OUTPUT: <__AArch64_backward_long_call_A_0>:
+# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+
+# CHECK-OUTPUT: <E>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 8bbc1b7bc85d1..9137b917950b4 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -69,8 +69,8 @@ hot:
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_long_call_foo>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_short_call_bar>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_long_call_foo_{{[0-9]+}}>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_short_call_bar_{{[0-9]+}}>
# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
.global _start
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 253da8072d6b8..1c79609b64d83 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -44,10 +44,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_backward_long_call_A,__AArch64_backward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D_0,__AArch64_backward_short_call_A_1,__AArch64_backward_long_call_A_0,__AArch64_backward_long_call_D_1 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_A,__AArch64_forward_short_call_D,__AArch64_backward_short_call_A,__AArch64_forward_long_call_D \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D_0,__AArch64_forward_short_call_A_1,__AArch64_backward_short_call_A_2,__AArch64_forward_long_call_D_0 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
@@ -175,60 +175,60 @@ D:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D_0>
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-OUTPUT: <__AArch64_forward_short_call_D>:
+# CHECK-OUTPUT: <__AArch64_forward_short_call_D_0>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
-# CHECK-OUTPUT: <__AArch64_backward_short_call_A>:
+# CHECK-OUTPUT: <__AArch64_backward_short_call_A_1>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-OUTPUT: <D>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A_1>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <D>
-# CHECK-OUTPUT: <__AArch64_backward_long_call_A>:
+# CHECK-OUTPUT: <__AArch64_backward_long_call_A_0>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
-# CHECK-OUTPUT: <__AArch64_backward_long_call_D>:
+# CHECK-OUTPUT: <__AArch64_backward_long_call_D_1>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_D>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_D_1>
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_A>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_A_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D_0>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_A>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_A_1>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D_0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
@@ -240,23 +240,23 @@ D:
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D_0>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D_0>
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_D>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_D_0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_short_call_A>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_short_call_A_2>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT: <D>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A_2>
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
index 9d2124fbfa40d..dd3e287b7da41 100644
--- a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
+++ b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
@@ -16,7 +16,7 @@
# RUN: --skip-funcs='^bar$' \
# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_long_call_bar \
+# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_long_call_bar_0 \
# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch \
@@ -94,9 +94,9 @@ padding:
#
# TAILCALL: <foo>:
# TAILCALL-NEXT: {{.*}} mov x16, #0x2a
-# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_long_call_bar>
+# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_long_call_bar_0>
#
-# TAILCALL: <__AArch64_forward_long_call_bar>:
+# TAILCALL: <__AArch64_forward_long_call_bar_0>:
# TAILCALL-NEXT: {{.*}} adrp x16,
# TAILCALL-NEXT: {{.*}} add x16, x16,
# TAILCALL-NEXT: {{.*}} br x16
>From 4a00843a8f683b99f326a89130bf46c65923d37e Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Fri, 11 Sep 2026 11:02:17 +0000
Subject: [PATCH 15/22] use branch chains instead of adrp if thunks needed <=
opts::MaxThunkChainLength
---
bolt/include/bolt/Passes/LongJmp.h | 62 ++-
bolt/lib/Passes/LongJmp.cpp | 440 +++++++++---------
.../AArch64/relax-branches-with-thunk-chain.s | 66 +--
.../AArch64/relax-call-long-thunk-reuse.s | 15 +-
.../AArch64/relax-call-short-thunk-reuse.s | 131 ++++++
bolt/test/AArch64/relax-calls.s | 28 +-
.../test/AArch64/relax-cross-fragment-calls.s | 67 +--
.../tail-classified-body-branch-live-x16.S | 16 +-
8 files changed, 488 insertions(+), 337 deletions(-)
create mode 100644 bolt/test/AArch64/relax-call-short-thunk-reuse.s
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index c119f511c0683..3a81fd54d5730 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -113,14 +113,15 @@ class LongJmpPass : public BinaryFunctionPass {
/// Thunks located before this cluster.
BinaryFunctionListType BackwardThunkList;
- /// Call thunks emitted by this cluster.
+ /// Long call thunks emitted by this cluster.
///
/// <Function Symbol> -> <Thunk Function>.
- DenseMap<const MCSymbol *, BinaryFunction *> CallThunks;
+ DenseMap<const MCSymbol *, BinaryFunction *> LongThunks;
+ /// B-only thunks emitted by this cluster.
+ ///
/// <Destination Symbol> -> <Thunk Function>.
- DenseMap<const MCSymbol *, BinaryFunction *> ForwardBranchThunks;
- DenseMap<const MCSymbol *, BinaryFunction *> BackwardBranchThunks;
+ DenseMap<const MCSymbol *, BinaryFunction *> BranchThunks;
StringRef getThunkSectionName(bool IsForward) const {
return IsForward ? EndSectionName : StartSectionName;
@@ -129,7 +130,7 @@ class LongJmpPass : public BinaryFunctionPass {
uint64_t getEndOffset() const { return StartOffset + Size; }
};
- struct FragmentClusterLayout {
+ struct ClusterLayout {
struct Position {
unsigned Cluster;
uint64_t Offset;
@@ -140,7 +141,7 @@ class LongJmpPass : public BinaryFunctionPass {
DenseMap<const MCSymbol *, Position> SymLayout;
};
- struct CrossClusterReference {
+ struct ClusterReference {
MCInst *Inst;
const MCSymbol *TargetSymbol;
uint64_t SourceOffset;
@@ -149,35 +150,52 @@ class LongJmpPass : public BinaryFunctionPass {
unsigned TargetCluster;
};
- struct ClusteredReferences {
- SmallVector<CrossClusterReference> ShortThunkCalls;
- SmallVector<CrossClusterReference> OutOfLayoutLongThunkCalls;
- SmallVector<SmallVector<CrossClusterReference>, 4> LongThunkCallsByDistance;
- SmallVector<CrossClusterReference> CrossClusterBranches;
+ struct ClusterRelaxationWorklist {
+ SmallVector<ClusterReference> OutOfLayoutCalls;
+ SmallVector<SmallVector<ClusterReference>, 4> CallsByDistance;
+ SmallVector<ClusterReference> Branches;
+ };
+
+ struct ClusterRelaxationStats {
+ size_t NumShortThunkCalls = 0;
+ size_t NumLongThunkCalls = 0;
+ size_t NumShortThunks = 0;
+ size_t NumShortThunksReused = 0;
+ size_t NumLongThunks = 0;
+ size_t NumLongThunksReused = 0;
+ size_t NumBranchThunks = 0;
+ size_t NumBranchThunksReused = 0;
};
- FragmentClusterLayout
+ ClusterLayout
buildClusterLayout(BinaryContext &BC,
const BinaryFunctionListType &OutputFunctions);
/// Collect call and branch references that need cluster-level relaxation.
- ClusteredReferences
- collectClusteredReferences(BinaryContext &BC,
- const BinaryFunctionListType &OutputFunctions,
- const FragmentClusterLayout &Layout);
+ ClusterRelaxationWorklist collectClusterRelaxationWorklist(
+ BinaryContext &BC, const BinaryFunctionListType &OutputFunctions,
+ const ClusterLayout &Layout);
+
+ /// Return the first thunk symbol for an unconditional branch thunk chain.
+ const MCSymbol *getOrCreateBranchThunkChain(BinaryContext &BC,
+ ClusterLayout &Layout,
+ ClusterRelaxationStats &Stats,
+ const ClusterReference &Ref,
+ unsigned MaxThunks);
/// Relax calls using function fragment clusters.
- void relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
- ClusteredReferences &References);
+ void relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
+ ClusterRelaxationWorklist &Worklist,
+ ClusterRelaxationStats &Stats);
/// Relax direct unconditional branches using function fragment clusters.
- void relaxUnconditionalBranches(BinaryContext &BC,
- FragmentClusterLayout &Layout,
- ClusteredReferences &References);
+ void relaxUnconditionalBranches(BinaryContext &BC, ClusterLayout &Layout,
+ ClusterRelaxationWorklist &Worklist,
+ ClusterRelaxationStats &Stats);
/// Insert all thunks owned by the fragment cluster layout.
void insertClusterThunks(BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout);
+ ClusterLayout &Layout);
/// Relax calls and direct unconditional branches using one cluster layout.
void relaxWithClusters(BinaryContext &BC);
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 95c44037904c1..b0b3cfbe59eeb 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -44,6 +44,11 @@ static cl::opt<unsigned long long> MaxClusterSize(
"max-cluster-size",
cl::desc("maximum estimated size of a function fragment cluster in bytes"),
cl::init(124 * 1024 * 1024), cl::cat(BoltOptCategory));
+
+static cl::opt<unsigned> MaxThunkChainLength(
+ "max-thunk-chain-length",
+ cl::desc("maximum number of B-only thunks to use for call relaxation"),
+ cl::init(1), cl::Hidden, cl::cat(BoltOptCategory));
}
namespace llvm {
@@ -1055,7 +1060,7 @@ static uint64_t estimateFragmentSize(const BinaryFunction &BF,
return Size;
}
-LongJmpPass::FragmentClusterLayout
+LongJmpPass::ClusterLayout
LongJmpPass::buildClusterLayout(BinaryContext &BC,
const BinaryFunctionListType &OutputFunctions) {
struct OutputFragment {
@@ -1070,7 +1075,7 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
size_t End;
};
- FragmentClusterLayout Layout;
+ ClusterLayout Layout;
SmallVector<SmallVector<OutputFragment>, 4> FragmentsBySection;
StringMap<size_t> SectionToBucket;
auto addOrderedFragment = [&](OutputFragment &&Fragment) {
@@ -1213,27 +1218,16 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
return Layout;
}
-LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
+LongJmpPass::ClusterRelaxationWorklist
+LongJmpPass::collectClusterRelaxationWorklist(
BinaryContext &BC, const BinaryFunctionListType &OutputFunctions,
- const FragmentClusterLayout &Layout) {
+ const ClusterLayout &Layout) {
auto &Clusters = Layout.Clusters;
auto &BBLayout = Layout.BBLayout;
auto &SymLayout = Layout.SymLayout;
- auto canUseShortThunk = [&](const CrossClusterReference &Call) {
- if (Call.TargetCluster == -1u)
- return false;
-
- const bool IsForward = Call.SourceCluster < Call.TargetCluster;
- const FragmentCluster &Source = Clusters[Call.SourceCluster];
- const uint64_t ThunkOffset =
- IsForward ? Source.getEndOffset() : Source.StartOffset;
- return isWithinClusterRange(Call.SourceOffset, ThunkOffset) &&
- isWithinClusterRange(ThunkOffset, Call.TargetOffset);
- };
-
- ClusteredReferences References;
- References.LongThunkCallsByDistance.resize(Clusters.size());
+ ClusterRelaxationWorklist Worklist;
+ Worklist.CallsByDistance.resize(Clusters.size());
auto isPrimaryEntryTarget = [&](const MCSymbol *TargetSymbol) {
uint64_t EntryID = 0;
const BinaryFunction *BF = BC.getFunctionForSymbol(TargetSymbol, &EntryID);
@@ -1248,7 +1242,7 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
auto SourceIt = BBLayout.find(&BB);
if (SourceIt == BBLayout.end())
continue;
- const FragmentClusterLayout::Position Source = SourceIt->second;
+ const ClusterLayout::Position Source = SourceIt->second;
uint64_t InstOffset = Source.Offset;
for (MCInst &Inst : BB) {
@@ -1269,12 +1263,11 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
auto TargetIt = SymLayout.find(TargetSymbol);
const bool Found = TargetIt != SymLayout.end();
// Unmapped targets use maximum offset and are treated as out of range.
- const FragmentClusterLayout::Position Target =
- Found ? TargetIt->second
- : FragmentClusterLayout::Position{-1u, -1ULL};
- const CrossClusterReference Reference{&Inst, TargetSymbol,
- SourceOffset, Target.Offset,
- Source.Cluster, Target.Cluster};
+ const ClusterLayout::Position Target =
+ Found ? TargetIt->second : ClusterLayout::Position{-1u, -1ULL};
+ const ClusterReference Reference{&Inst, TargetSymbol,
+ SourceOffset, Target.Offset,
+ Source.Cluster, Target.Cluster};
const bool UseBranchChain =
Found && (IsUncondBranch ||
(IsTailCall && !isPrimaryEntryTarget(TargetSymbol)));
@@ -1289,7 +1282,7 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
if (isWithinClusterRange(SourceOffset, Target.Offset))
continue;
- References.CrossClusterBranches.push_back(Reference);
+ Worklist.Branches.push_back(Reference);
continue;
}
@@ -1301,99 +1294,181 @@ LongJmpPass::ClusteredReferences LongJmpPass::collectClusteredReferences(
if (isWithinClusterRange(SourceOffset, Target.Offset))
continue;
- if (canUseShortThunk(Reference)) {
- References.ShortThunkCalls.push_back(Reference);
- } else if (Reference.TargetCluster == -1u) {
- References.OutOfLayoutLongThunkCalls.push_back(Reference);
+ if (Reference.TargetCluster == -1u) {
+ Worklist.OutOfLayoutCalls.push_back(Reference);
} else {
const unsigned Distance = getClusterDistance(Reference.SourceCluster,
Reference.TargetCluster);
- References.LongThunkCallsByDistance[Distance].push_back(Reference);
+ Worklist.CallsByDistance[Distance].push_back(Reference);
}
}
}
}
- return References;
+ return Worklist;
}
-void LongJmpPass::relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
- ClusteredReferences &References) {
+const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
+ BinaryContext &BC, ClusterLayout &Layout, ClusterRelaxationStats &Stats,
+ const ClusterReference &Ref, unsigned MaxThunks) {
auto &Clusters = Layout.Clusters;
- auto &ShortThunkCalls = References.ShortThunkCalls;
- auto &OutOfLayoutLongThunkCalls = References.OutOfLayoutLongThunkCalls;
- auto &LongThunkCallsByDistance = References.LongThunkCallsByDistance;
-
- size_t NumShortThunks = 0;
- size_t NumLongThunks = 0;
- size_t NumShortThunksReused = 0;
- size_t NumLongThunksReused = 0;
- auto createCallThunk = [&](const MCSymbol *TargetSymbol, bool IsShort,
- bool IsForward) {
- BinaryFunction *Thunk = nullptr;
- const size_t ThunkNumber = IsShort ? NumShortThunks++ : NumLongThunks++;
+ const unsigned SourceCluster = Ref.SourceCluster;
+ const unsigned TargetCluster = Ref.TargetCluster;
+ const bool IsForward = SourceCluster < TargetCluster;
+ const bool IsCall = BC.MIB->isCall(*Ref.Inst);
+ const unsigned NumHops = getClusterDistance(SourceCluster, TargetCluster);
+
+ auto createBranchThunk = [&](const MCSymbol *TargetSymbol) {
+ const size_t ThunkNumber =
+ IsCall ? Stats.NumShortThunks++ : Stats.NumBranchThunks++;
std::string ThunkName =
(Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
- (IsShort ? "short_call_" : "long_call_") + TargetSymbol->getName() +
- "_" + Twine(ThunkNumber))
+ "Thunk_" + (IsCall ? Twine(Ref.TargetSymbol->getName()) + "_" : "") +
+ Twine(ThunkNumber))
.str();
- if (IsShort) {
- Thunk = BC.createThunkBinaryFunction(ThunkName);
- MCInst Inst;
- BC.MIB->createTailCall(Inst, TargetSymbol, BC.Ctx.get());
- Thunk->addBasicBlock()->addInstruction(Inst);
- } else {
- Thunk = BC.createThunkBinaryFunction(ThunkName);
- InstructionListType Instructions;
- BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
- Thunk->addBasicBlock()->addInstructions(Instructions);
+
+ BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(ThunkName);
+ MCInst Inst;
+ BC.MIB->createUncondBranch(Inst, TargetSymbol, BC.Ctx.get());
+ ThunkBF->addBasicBlock()->addInstruction(Inst);
+ return ThunkBF;
+ };
+
+ auto registerBranchThunk = [&](FragmentCluster &Cluster,
+ BinaryFunction *Thunk) {
+ uint64_t EntryID = 0;
+ const BinaryFunction *BF =
+ IsCall ? BC.getFunctionForSymbol(Ref.TargetSymbol, &EntryID) : nullptr;
+ if (BF && EntryID == 0)
+ for (const MCSymbol *Symbol : BF->getSymbols())
+ Cluster.BranchThunks[Symbol] = Thunk;
+ else
+ Cluster.BranchThunks[Ref.TargetSymbol] = Thunk;
+ };
+
+ auto getOrCreateBranchThunk = [&](FragmentCluster &Cluster,
+ const MCSymbol *NextTarget) {
+ auto It = Cluster.BranchThunks.find(Ref.TargetSymbol);
+ if (It != Cluster.BranchThunks.end()) {
+ if (IsCall)
+ ++Stats.NumShortThunksReused;
+ else
+ ++Stats.NumBranchThunksReused;
+ return It->second;
}
+
+ BinaryFunction *Thunk = createBranchThunk(NextTarget);
+ Thunk->setCodeSectionName(Cluster.getThunkSectionName(IsForward));
+ auto &ThunkList =
+ IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
+ ThunkList.push_back(Thunk);
+ registerBranchThunk(Cluster, Thunk);
return Thunk;
};
- auto registerCallThunk = [&](FragmentCluster &FC, const MCSymbol *Callee,
+ auto getClusterAtHop = [&](const unsigned Hop) {
+ return IsForward ? SourceCluster + Hop : SourceCluster - Hop;
+ };
+
+ auto getThunkOffset = [&](const unsigned Cluster) {
+ const FragmentCluster &FC = Clusters[Cluster];
+ return IsForward ? FC.getEndOffset() : FC.StartOffset;
+ };
+
+ SmallVector<unsigned> ThunkClusters;
+ uint64_t CurrentOffset = Ref.SourceOffset;
+ unsigned NextHop = 0;
+ while (!isWithinClusterRange(CurrentOffset, Ref.TargetOffset)) {
+ if (ThunkClusters.size() == MaxThunks)
+ return nullptr;
+
+ unsigned BestHop = -1u;
+ for (unsigned Hop = NextHop; Hop < NumHops; ++Hop) {
+ const unsigned Cluster = getClusterAtHop(Hop);
+ if (!isWithinClusterRange(CurrentOffset, getThunkOffset(Cluster)))
+ break;
+ BestHop = Hop;
+ }
+
+ if (BestHop == -1u)
+ return nullptr;
+
+ const unsigned BestCluster = getClusterAtHop(BestHop);
+ ThunkClusters.push_back(BestCluster);
+ CurrentOffset = getThunkOffset(BestCluster);
+ NextHop = BestHop + 1;
+ }
+
+ BinaryFunction *FirstThunk = nullptr;
+ const MCSymbol *NextTarget = Ref.TargetSymbol;
+ for (const unsigned Cluster : llvm::reverse(ThunkClusters)) {
+ FirstThunk = getOrCreateBranchThunk(Clusters[Cluster], NextTarget);
+ NextTarget = FirstThunk->getSymbol();
+ }
+
+ return FirstThunk ? FirstThunk->getSymbol() : Ref.TargetSymbol;
+}
+
+void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
+ ClusterRelaxationWorklist &Worklist,
+ ClusterRelaxationStats &Stats) {
+ auto &Clusters = Layout.Clusters;
+ auto &OutOfLayoutCalls = Worklist.OutOfLayoutCalls;
+ auto &CallsByDistance = Worklist.CallsByDistance;
+
+ auto createLongThunk = [&](const MCSymbol *TargetSymbol, bool IsForward) {
+ BinaryFunction *Thunk = nullptr;
+ const size_t ThunkNumber = Stats.NumLongThunks++;
+ std::string ThunkName =
+ (Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
+ "ADRPThunk_" + TargetSymbol->getName() + "_" + Twine(ThunkNumber))
+ .str();
+ Thunk = BC.createThunkBinaryFunction(ThunkName);
+ InstructionListType Instructions;
+ BC.MIB->createLongTailCall(Instructions, TargetSymbol, BC.Ctx.get());
+ Thunk->addBasicBlock()->addInstructions(Instructions);
+ return Thunk;
+ };
+
+ auto registerLongThunk = [&](FragmentCluster &FC, const MCSymbol *Callee,
BinaryFunction *Thunk) {
uint64_t EntryID = 0;
const BinaryFunction *BF = BC.getFunctionForSymbol(Callee, &EntryID);
const bool IsPrimaryEntry = EntryID == 0;
if (BF && IsPrimaryEntry)
for (const MCSymbol *Symbol : BF->getSymbols())
- FC.CallThunks[Symbol] = Thunk;
+ FC.LongThunks[Symbol] = Thunk;
else
- FC.CallThunks[Callee] = Thunk;
+ FC.LongThunks[Callee] = Thunk;
};
- auto getOrCreateCallThunk = [&](const MCSymbol *TargetSymbol,
- unsigned SourceCluster, bool IsShort,
- bool IsForward) {
+ auto getOrCreateLongThunk = [&](const ClusterReference &Call) {
+ const MCSymbol *TargetSymbol = Call.TargetSymbol;
+ const unsigned SourceCluster = Call.SourceCluster;
+ const bool IsForward = SourceCluster < Call.TargetCluster;
FragmentCluster &FC = Clusters[SourceCluster];
- if (auto It = FC.CallThunks.find(TargetSymbol); It != FC.CallThunks.end()) {
- if (IsShort)
- ++NumShortThunksReused;
- else
- ++NumLongThunksReused;
+ if (auto It = FC.LongThunks.find(TargetSymbol); It != FC.LongThunks.end()) {
+ ++Stats.NumLongThunksReused;
return It->second;
}
// Reuse long thunks hosted at the adjacent cluster boundary.
- if (!IsShort) {
- FragmentCluster *ReuseCluster = nullptr;
- if (IsForward && SourceCluster > 0)
- ReuseCluster = &Clusters[SourceCluster - 1];
- else if (!IsForward && SourceCluster + 1 < Clusters.size())
- ReuseCluster = &Clusters[SourceCluster + 1];
-
- if (ReuseCluster) {
- auto It = ReuseCluster->CallThunks.find(TargetSymbol);
- if (It != ReuseCluster->CallThunks.end()) {
- BinaryFunction *Thunk = It->second;
- ++NumLongThunksReused;
- return Thunk;
- }
+ FragmentCluster *ReuseCluster = nullptr;
+ if (IsForward && SourceCluster > 0)
+ ReuseCluster = &Clusters[SourceCluster - 1];
+ else if (!IsForward && SourceCluster + 1 < Clusters.size())
+ ReuseCluster = &Clusters[SourceCluster + 1];
+
+ if (ReuseCluster) {
+ auto It = ReuseCluster->LongThunks.find(TargetSymbol);
+ if (It != ReuseCluster->LongThunks.end()) {
+ BinaryFunction *Thunk = It->second;
+ ++Stats.NumLongThunksReused;
+ return Thunk;
}
}
- BinaryFunction *Thunk = createCallThunk(TargetSymbol, IsShort, IsForward);
+ BinaryFunction *Thunk = createLongThunk(TargetSymbol, IsForward);
Thunk->setCodeSectionName(FC.getThunkSectionName(IsForward));
BinaryFunctionListType &ThunkList =
@@ -1401,164 +1476,88 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, FragmentClusterLayout &Layout,
ThunkList.push_back(Thunk);
// Register thunks for all symbols associated with the function.
- registerCallThunk(FC, TargetSymbol, Thunk);
+ registerLongThunk(FC, TargetSymbol, Thunk);
return Thunk;
};
- auto relaxCall = [&](CrossClusterReference &Call, bool IsShort) {
- const bool IsForward = Call.SourceCluster < Call.TargetCluster;
- BinaryFunction *Thunk = getOrCreateCallThunk(
- Call.TargetSymbol, Call.SourceCluster, IsShort, IsForward);
+ auto relaxCallWithLongThunk = [&](ClusterReference &Call) {
+ BinaryFunction *Thunk = getOrCreateLongThunk(Call);
BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
};
- for (CrossClusterReference &Call : ShortThunkCalls)
- relaxCall(Call, /*IsShort=*/true);
-
- // Process out-of-layout targets first, then known targets from farthest to
- // nearest, so closer calls can reuse long thunks at cluster boundaries.
- for (CrossClusterReference &Call : OutOfLayoutLongThunkCalls)
- relaxCall(Call, /*IsShort=*/false);
-
- size_t NumLongThunkCalls = OutOfLayoutLongThunkCalls.size();
- for (auto &Calls : llvm::reverse(LongThunkCallsByDistance)) {
- NumLongThunkCalls += Calls.size();
- for (CrossClusterReference &Call : Calls)
- relaxCall(Call, /*IsShort=*/false);
+ // Process out-of-layout targets first.
+ for (ClusterReference &Call : OutOfLayoutCalls)
+ relaxCallWithLongThunk(Call);
+
+ Stats.NumLongThunkCalls += OutOfLayoutCalls.size();
+ // Process known targets from farthest to nearest for maximizing thunk reuse.
+ for (auto &Calls : llvm::reverse(CallsByDistance)) {
+ for (ClusterReference &Call : Calls) {
+ // Attempt a branch thunk chain if length is below threshold.
+ if (const MCSymbol *Target = getOrCreateBranchThunkChain(
+ BC, Layout, Stats, Call, opts::MaxThunkChainLength)) {
+ BC.MIB->replaceBranchTarget(*Call.Inst, Target, BC.Ctx.get());
+ ++Stats.NumShortThunkCalls;
+ continue;
+ }
+ // Otherwise fall back to a long thunk.
+ ++Stats.NumLongThunkCalls;
+ relaxCallWithLongThunk(Call);
+ }
}
- if (!ShortThunkCalls.empty())
- BC.outs() << "BOLT-INFO: relaxed " << ShortThunkCalls.size()
- << " short cluster calls with thunks\n";
+ if (Stats.NumShortThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << Stats.NumShortThunkCalls
+ << " calls with short thunks\n";
- if (NumLongThunkCalls)
- BC.outs() << "BOLT-INFO: relaxed " << NumLongThunkCalls
- << " long cluster calls with thunks\n";
+ if (Stats.NumLongThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << Stats.NumLongThunkCalls
+ << " calls with long thunks\n";
- if (NumShortThunks)
- BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
+ if (Stats.NumShortThunks)
+ BC.outs() << "BOLT-INFO: " << Stats.NumShortThunks
+ << " short thunks created\n";
- if (NumLongThunks)
- BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
+ if (Stats.NumLongThunks)
+ BC.outs() << "BOLT-INFO: " << Stats.NumLongThunks
+ << " long thunks created\n";
- if (NumShortThunksReused)
- BC.outs() << "BOLT-INFO: " << NumShortThunksReused
+ if (Stats.NumShortThunksReused)
+ BC.outs() << "BOLT-INFO: " << Stats.NumShortThunksReused
<< " short thunks reused\n";
- if (NumLongThunksReused)
- BC.outs() << "BOLT-INFO: " << NumLongThunksReused
+ if (Stats.NumLongThunksReused)
+ BC.outs() << "BOLT-INFO: " << Stats.NumLongThunksReused
<< " long thunks reused\n";
}
-void LongJmpPass::relaxUnconditionalBranches(BinaryContext &BC,
- FragmentClusterLayout &Layout,
- ClusteredReferences &References) {
- auto &Clusters = Layout.Clusters;
- auto &CrossClusterBranches = References.CrossClusterBranches;
-
- size_t NumBranchThunks = 0;
- size_t NumBranchThunksReused = 0;
- auto createBranchThunk = [&](const MCSymbol *TargetSymbol,
- const bool IsForward) {
- std::string ThunkName =
- (Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
- "branch_chain_" + Twine(NumBranchThunks++))
- .str();
-
- BinaryFunction *ThunkBF = BC.createThunkBinaryFunction(ThunkName);
- MCInst Inst;
- BC.MIB->createUncondBranch(Inst, TargetSymbol, BC.Ctx.get());
- ThunkBF->addBasicBlock()->addInstruction(Inst);
- return ThunkBF;
- };
-
- auto getOrCreateBranchThunk =
- [&](FragmentCluster &Cluster, const MCSymbol *TargetSymbol,
- const MCSymbol *NextTarget, const bool IsForward) {
- auto &Thunks = IsForward ? Cluster.ForwardBranchThunks
- : Cluster.BackwardBranchThunks;
- auto It = Thunks.find(TargetSymbol);
- if (It != Thunks.end()) {
- ++NumBranchThunksReused;
- return It->second;
- }
-
- BinaryFunction *Thunk = createBranchThunk(NextTarget, IsForward);
- Thunk->setCodeSectionName(Cluster.getThunkSectionName(IsForward));
- auto &ThunkList =
- IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
- ThunkList.push_back(Thunk);
- Thunks[TargetSymbol] = Thunk;
- return Thunk;
- };
-
- auto getOrCreateBranchThunkChain =
- [&](const CrossClusterReference &Branch) -> const MCSymbol * {
- const unsigned SourceCluster = Branch.SourceCluster;
- const unsigned TargetCluster = Branch.TargetCluster;
- const bool IsForward = SourceCluster < TargetCluster;
- const unsigned NumHops = IsForward ? TargetCluster - SourceCluster
- : SourceCluster - TargetCluster;
-
- auto getClusterAtHop = [&](const unsigned Hop) {
- return IsForward ? SourceCluster + Hop : SourceCluster - Hop;
- };
-
- auto getThunkOffset = [&](const unsigned Cluster) {
- const FragmentCluster &FC = Clusters[Cluster];
- return IsForward ? FC.getEndOffset() : FC.StartOffset;
- };
-
- SmallVector<unsigned> ThunkClusters;
- uint64_t CurrentOffset = Branch.SourceOffset;
- unsigned NextHop = 0;
- while (!isWithinClusterRange(CurrentOffset, Branch.TargetOffset)) {
- unsigned BestHop = -1u;
- for (unsigned Hop = NextHop; Hop < NumHops; ++Hop) {
- const unsigned Cluster = getClusterAtHop(Hop);
- if (!isWithinClusterRange(CurrentOffset, getThunkOffset(Cluster)))
- break;
- BestHop = Hop;
- }
-
- assert(BestHop != -1u && "expected reachable branch thunk");
- const unsigned BestCluster = getClusterAtHop(BestHop);
- ThunkClusters.push_back(BestCluster);
- CurrentOffset = getThunkOffset(BestCluster);
- NextHop = BestHop + 1;
- }
-
- BinaryFunction *FirstThunk = nullptr;
- const MCSymbol *NextTarget = Branch.TargetSymbol;
- for (const unsigned Cluster : llvm::reverse(ThunkClusters)) {
- FirstThunk = getOrCreateBranchThunk(
- Clusters[Cluster], Branch.TargetSymbol, NextTarget, IsForward);
- NextTarget = FirstThunk->getSymbol();
- }
-
- assert(FirstThunk && "expected branch thunk chain");
- return FirstThunk->getSymbol();
- };
+void LongJmpPass::relaxUnconditionalBranches(
+ BinaryContext &BC, ClusterLayout &Layout,
+ ClusterRelaxationWorklist &Worklist, ClusterRelaxationStats &Stats) {
+ auto &Branches = Worklist.Branches;
- for (const CrossClusterReference &Branch : CrossClusterBranches) {
- const MCSymbol *Target = getOrCreateBranchThunkChain(Branch);
+ for (const ClusterReference &Branch : Branches) {
+ const MCSymbol *Target = getOrCreateBranchThunkChain(
+ BC, Layout, Stats, Branch, /*MaxThunks=*/-1u);
+ assert(Target && "expected branch thunk chain");
BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
}
- if (!CrossClusterBranches.empty())
- BC.outs() << "BOLT-INFO: relaxed " << CrossClusterBranches.size()
- << " cross-cluster branches\n";
+ if (!Branches.empty())
+ BC.outs() << "BOLT-INFO: relaxed " << Branches.size()
+ << " unconditional branches\n";
- if (NumBranchThunks)
- BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
+ if (Stats.NumBranchThunks)
+ BC.outs() << "BOLT-INFO: " << Stats.NumBranchThunks
+ << " branch thunks created\n";
- if (NumBranchThunksReused)
- BC.outs() << "BOLT-INFO: " << NumBranchThunksReused
+ if (Stats.NumBranchThunksReused)
+ BC.outs() << "BOLT-INFO: " << Stats.NumBranchThunksReused
<< " branch thunks reused\n";
}
void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
- FragmentClusterLayout &Layout) {
+ ClusterLayout &Layout) {
struct ThunkInsertion {
size_t Position;
bool IsForward;
@@ -1594,15 +1593,16 @@ void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
- FragmentClusterLayout Layout = buildClusterLayout(BC, OutputFunctions);
+ ClusterLayout Layout = buildClusterLayout(BC, OutputFunctions);
if (Layout.Clusters.empty())
return;
- ClusteredReferences References =
- collectClusteredReferences(BC, OutputFunctions, Layout);
- relaxCalls(BC, Layout, References);
- relaxUnconditionalBranches(BC, Layout, References);
+ ClusterRelaxationWorklist Worklist =
+ collectClusterRelaxationWorklist(BC, OutputFunctions, Layout);
+ ClusterRelaxationStats Stats;
+ relaxCalls(BC, Layout, Worklist, Stats);
+ relaxUnconditionalBranches(BC, Layout, Worklist, Stats);
insertClusterThunks(OutputFunctions, Layout);
LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index b376ae570f149..e8699ffe33d20 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -30,10 +30,10 @@
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_0,__AArch64_forward_branch_chain_2,__AArch64_forward_branch_chain_3,__AArch64_forward_branch_chain_6,__AArch64_backward_branch_chain_1,__AArch64_backward_branch_chain_4,__AArch64_backward_branch_chain_5,__AArch64_backward_branch_chain_7 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_Thunk_0,__AArch64_forward_Thunk_2,__AArch64_forward_Thunk_3,__AArch64_forward_Thunk_6,__AArch64_backward_Thunk_1,__AArch64_backward_Thunk_4,__AArch64_backward_Thunk_5,__AArch64_backward_Thunk_7 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_branch_chain_1,__AArch64_forward_branch_chain_4,__AArch64_forward_branch_chain_5,__AArch64_backward_branch_chain_0,__AArch64_backward_branch_chain_2,__AArch64_backward_branch_chain_3 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_Thunk_1,__AArch64_forward_Thunk_4,__AArch64_forward_Thunk_5,__AArch64_backward_Thunk_0,__AArch64_backward_Thunk_2,__AArch64_backward_Thunk_3 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
@@ -46,7 +46,7 @@
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes
-# CHECK-BOLT: BOLT-INFO: relaxed 7 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: relaxed 7 unconditional branches
# CHECK-BOLT: BOLT-INFO: 8 branch thunks created
# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
@@ -60,7 +60,7 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 unconditional branches
# CHECK-BOLT-HFE: BOLT-INFO: 6 branch thunks created
# CHECK-SECTIONS: .text
@@ -179,62 +179,62 @@ pad_hot_3:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[A_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[A_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
+# CHECK-OUTPUT-NEXT: [[A_BR]]: {{.*}} b 0x[[A_FW0:[0-9a-f]+]] <__AArch64_forward_Thunk_0>
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} tbz w0, #0x0, 0x[[B_ALT:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: {{.*}} cbz x0, 0x[[B_RET:[0-9a-f]+]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_3>
-# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64_forward_branch_chain_3>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_FW0:[0-9a-f]+]] <__AArch64_forward_Thunk_3>
+# CHECK-OUTPUT-NEXT: [[B_ALT]]: {{.*}} b 0x[[B_FW0]] <__AArch64_forward_Thunk_3>
# CHECK-OUTPUT-NEXT: [[B_RET]]: {{.*}} ret
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_0>:
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_0>:
# CHECK-OUTPUT-NEXT: [[A_FW0]]: {{.*}} b 0x[[A_COLD:[0-9a-f]+]] <A.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_3>:
-# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_2>
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_3>:
+# CHECK-OUTPUT-NEXT: [[B_FW0]]: {{.*}} b 0x[[B_FW1:[0-9a-f]+]] <__AArch64_forward_Thunk_2>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_1>:
+# CHECK-OUTPUT: <__AArch64_backward_Thunk_1>:
# CHECK-OUTPUT-NEXT: [[A_BW0:[0-9a-f]+]]: {{.*}} b 0x[[A_RET]] <A+0x4>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_4>:
+# CHECK-OUTPUT: <__AArch64_backward_Thunk_4>:
# CHECK-OUTPUT-NEXT: [[B_BW1:[0-9a-f]+]]: {{.*}} b 0x[[B_RET]] <B+0x10>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[C_RET:[0-9a-f]+]]: {{.*}} ret
-# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_6>
+# CHECK-OUTPUT-NEXT: [[C_BR]]: {{.*}} b 0x[[C_FW0:[0-9a-f]+]] <__AArch64_forward_Thunk_6>
# CHECK-OUTPUT: <D>:
# CHECK-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-OUTPUT-NEXT: [[D_RET:[0-9a-f]+]]: {{.*}} ret
# CHECK-OUTPUT-NEXT: [[D_BR]]: {{.*}} b 0x[[D_COLD:[0-9a-f]+]] <D.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_2>:
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_2>:
# CHECK-OUTPUT-NEXT: [[B_FW1]]: {{.*}} b 0x[[B_COLD:[0-9a-f]+]] <B.cold.0>
-# CHECK-OUTPUT: <__AArch64_forward_branch_chain_6>:
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_6>:
# CHECK-OUTPUT-NEXT: [[C_FW0]]: {{.*}} b 0x[[C_COLD:[0-9a-f]+]] <C.cold.0>
# CHECK-OUTPUT: Disassembly of section .text.cold:
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_5>:
-# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64_backward_branch_chain_4>
+# CHECK-OUTPUT: <__AArch64_backward_Thunk_5>:
+# CHECK-OUTPUT-NEXT: [[B_BW0:[0-9a-f]+]]: {{.*}} b 0x[[B_BW1]] <__AArch64_backward_Thunk_4>
-# CHECK-OUTPUT: <__AArch64_backward_branch_chain_7>:
+# CHECK-OUTPUT: <__AArch64_backward_Thunk_7>:
# CHECK-OUTPUT-NEXT: [[C_BW0:[0-9a-f]+]]: {{.*}} b 0x[[C_RET]] <C+0x4>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT-NEXT: [[A_COLD]]: {{.*}} mov x0, #0x1
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_branch_chain_1>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[A_BW0]] <__AArch64_backward_Thunk_1>
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: [[B_COLD]]: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64_backward_branch_chain_5>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[B_BW0]] <__AArch64_backward_Thunk_5>
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: [[C_COLD]]: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64_backward_branch_chain_7>
+# CHECK-OUTPUT-NEXT: {{.*}} b 0x[[C_BW0]] <__AArch64_backward_Thunk_7>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: [[D_COLD]]: {{.*}} mov x0, #0x4
@@ -253,24 +253,24 @@ pad_hot_3:
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x3
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_1>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_C_FW0:[0-9a-f]+]] <__AArch64_forward_Thunk_1>
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_COLD:[0-9a-f]+]]: {{.*}} mov x0, #0x4
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_branch_chain_5>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b 0x[[HFE_D_FW0:[0-9a-f]+]] <__AArch64_forward_Thunk_5>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_1>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_Thunk_1>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_FW0]]: {{.*}} b 0x[[HFE_C_RET:[0-9a-f]+]] <C+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_5>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_branch_chain_4>
+# CHECK-HFE-OUTPUT: <__AArch64_forward_Thunk_5>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW0]]: {{.*}} b 0x[[HFE_D_FW1:[0-9a-f]+]] <__AArch64_forward_Thunk_4>
# CHECK-HFE-OUTPUT: Disassembly of section .text:
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_0>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_Thunk_0>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_C_COLD]] <C.cold.0>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_2>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_Thunk_2>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW1:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_COLD]] <D.cold.0>
# CHECK-HFE-OUTPUT: <A>:
@@ -285,18 +285,18 @@ pad_hot_3:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_ALT]]: {{.*}} b 0x[[HFE_B_COLD]] <B.cold.0>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_B_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT: <__AArch64_forward_branch_chain_4>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_Thunk_4>:
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_FW1]]: {{.*}} b 0x[[HFE_D_RET:[0-9a-f]+]] <D+0x4>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_branch_chain_3>:
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_branch_chain_2>
+# CHECK-HFE-OUTPUT: <__AArch64_backward_Thunk_3>:
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BW0:[0-9a-f]+]]: {{.*}} b 0x[[HFE_D_BW1]] <__AArch64_backward_Thunk_2>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_C_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_branch_chain_0>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_C_BR]]: {{.*}} b 0x[[HFE_C_BW1]] <__AArch64_backward_Thunk_0>
# CHECK-HFE-OUTPUT: <D>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} cbnz x0, 0x[[HFE_D_BR:[0-9a-f]+]] <{{.*}}>
# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_RET]]: {{.*}} ret
-# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_branch_chain_3>
+# CHECK-HFE-OUTPUT-NEXT: [[HFE_D_BR]]: {{.*}} b 0x[[HFE_D_BW0]] <__AArch64_backward_Thunk_3>
diff --git a/bolt/test/AArch64/relax-call-long-thunk-reuse.s b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
index 1942b55c38202..d7436daff83ec 100644
--- a/bolt/test/AArch64/relax-call-long-thunk-reuse.s
+++ b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
@@ -21,13 +21,14 @@
# RUN: llvm-strip --strip-unneeded %t
# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata \
# RUN: --compact-code-model --relax-exp --max-cluster-size=64 \
+# RUN: --max-thunk-chain-length=0 \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,E,__AArch64_backward_long_call_A_0,__AArch64_backward_long_call_A_1 \
+# RUN: --disassemble-symbols=A,B,C,D,E,__AArch64_backward_ADRPThunk_A_0,__AArch64_backward_ADRPThunk_A_1 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 5 function fragment cluster(s)
-# CHECK-BOLT: BOLT-INFO: relaxed 3 long cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 3 calls with long thunks
# CHECK-BOLT: BOLT-INFO: 2 long thunks created
# CHECK-BOLT: BOLT-INFO: 1 long thunks reused
@@ -89,21 +90,21 @@ E:
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} ret
-# CHECK-OUTPUT: <__AArch64_backward_long_call_A_1>:
+# CHECK-OUTPUT: <__AArch64_backward_ADRPThunk_A_1>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
# CHECK-OUTPUT: <C>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_1>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_1>
# CHECK-OUTPUT: <D>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_0>
-# CHECK-OUTPUT: <__AArch64_backward_long_call_A_0>:
+# CHECK-OUTPUT: <__AArch64_backward_ADRPThunk_A_0>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
# CHECK-OUTPUT: <E>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_0>
diff --git a/bolt/test/AArch64/relax-call-short-thunk-reuse.s b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
new file mode 100644
index 0000000000000..54a5b0bf4da6c
--- /dev/null
+++ b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
@@ -0,0 +1,131 @@
+## Check that short call thunks are reused. With --max-cluster-size=160,
+## A/B form the source cluster, C/D/E separate the target cluster, and F is the
+## call target. A builds a three-hop thunk chain. B reuses the whole chain,
+## while C and D reuse the rungs already placed after their clusters.
+##
+## cluster 0: A -> F, B -> F
+## cluster 1: C -> F
+## cluster 2: D -> F
+## cluster 3: E
+## cluster 4: F
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-strip --strip-unneeded %t
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata \
+# RUN: --compact-code-model --relax-exp --max-cluster-size=160 \
+# RUN: --max-thunk-chain-length=3 \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=A,B,C,D,E,F,__AArch64_forward_Thunk_F_0,__AArch64_forward_Thunk_F_1,__AArch64_forward_Thunk_F_2 \
+# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+
+# CHECK-BOLT: BOLT-INFO: built 5 function fragment cluster(s)
+# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with short thunks
+# CHECK-BOLT: BOLT-INFO: 3 short thunks created
+# CHECK-BOLT: BOLT-INFO: 6 short thunks reused
+
+## The reuse counter is per reused chain rung:
+##
+## B -> F reuses cluster 0, cluster 1, cluster 2 thunks = 3
+## C -> F reuses cluster 1, cluster 2 thunks = 2
+## D -> F reuses cluster 2 thunk = 1
+##
+## Total reused thunk lookups:
+##
+## 3 + 2 + 1 = 6
+
+ .text
+ .globl A
+ .type A, %function
+A:
+.A_entry:
+# FDATA: 1 A #.A_entry# 100
+ bl F
+ ret
+ .space 0x30
+ .size A, .-A
+
+ .globl B
+ .type B, %function
+B:
+.B_entry:
+# FDATA: 1 B #.B_entry# 100
+ bl F
+ ret
+ .space 0x30
+ .size B, .-B
+
+ .globl C
+ .type C, %function
+C:
+.C_entry:
+# FDATA: 1 C #.C_entry# 100
+ bl F
+ ret
+ .space 0x60
+ .size C, .-C
+
+ .globl D
+ .type D, %function
+D:
+.D_entry:
+# FDATA: 1 D #.D_entry# 100
+ bl F
+ ret
+ .space 0x60
+ .size D, .-D
+
+ .globl E
+ .type E, %function
+E:
+.E_entry:
+# FDATA: 1 E #.E_entry# 100
+ ret
+ .space 0x80
+ .size E, .-E
+
+ .globl F
+ .type F, %function
+F:
+.F_entry:
+# FDATA: 1 F #.F_entry# 100
+ ret
+ .space 0x30
+ .size F, .-F
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# CHECK-OUTPUT: <A>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_F_2>
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <B>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_F_2>
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_F_2>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <__AArch64_forward_Thunk_F_1>
+
+# CHECK-OUTPUT: <C>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_F_1>
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_F_1>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <__AArch64_forward_Thunk_F_0>
+
+# CHECK-OUTPUT: <D>:
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_F_0>
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_F_0>:
+# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <F>
+
+# CHECK-OUTPUT: <E>:
+# CHECK-OUTPUT-NEXT: {{.*}} ret
+
+# CHECK-OUTPUT: <F>:
+# CHECK-OUTPUT-NEXT: {{.*}} ret
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index 9137b917950b4..ec89f3ce69f85 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -6,12 +6,15 @@
# RUN: link_fdata %s %t.o %t.fdata
# RUN: llvm-strip --strip-unneeded %t.o
# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=1 --data %t.fdata \
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
+# RUN: --lite=1 --data %t.fdata \
# RUN: --print-normalized 2>&1 | FileCheck %s --check-prefix=CHECK-BOLT-LITE
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=0 --data %t.fdata \
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
+# RUN: --lite=0 --data %t.fdata \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --hot-functions-at-end --lite=0 \
-# RUN: --data %t.fdata | FileCheck %s --check-prefix=CHECK-BOLT-HOT-END
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
+# RUN: --hot-functions-at-end --lite=0 --data %t.fdata \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HOT-END
# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
## Constant islands at the end of functions foo(), bar(), and _start() make each
@@ -55,22 +58,19 @@ hot:
# CHECK-BOLT-LITE: BOLT-INFO: 3 long thunks created
## Check the number of thunks created in other modes.
-# CHECK-BOLT: BOLT-INFO: relaxed 2 short cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: relaxed 2 long cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: 1 short thunks created
-# CHECK-BOLT: BOLT-INFO: 2 long thunks created
+# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with short thunks
+# CHECK-BOLT: BOLT-INFO: 3 short thunks created
# CHECK-BOLT: BOLT-INFO: 1 short thunks reused
-# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 2 short cluster calls with thunks
-# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 2 long cluster calls with thunks
-# CHECK-BOLT-HOT-END: BOLT-INFO: 2 short thunks created
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 3 calls with short thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: relaxed 1 calls with long thunks
+# CHECK-BOLT-HOT-END: BOLT-INFO: 3 short thunks created
# CHECK-BOLT-HOT-END: BOLT-INFO: 1 long thunks created
-# CHECK-BOLT-HOT-END: BOLT-INFO: 1 long thunks reused
## Check that correct veneers are used depending on the target proximity.
# CHECK-OUTPUT-LABEL: <hot>:
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_long_call_foo_{{[0-9]+}}>
-# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_short_call_bar_{{[0-9]+}}>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_ADRPThunk_foo_{{[0-9]+}}>
+# CHECK-OUTPUT-NEXT: bl {{.*}} <__AArch64_backward_Thunk_bar_{{[0-9]+}}>
# CHECK-OUTPUT-NEXT: bl {{.*}} <_start>
.global _start
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 1c79609b64d83..21075d979dcd5 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -1,9 +1,9 @@
## Relax direct calls using thunks across function fragment clusters. A/B/C/D
## contain 42MiB islands. With --split-functions, BOLT places the cold blocks
## of A/B/C/D in .text.cold. The test exercises direct adjacent-cluster calls,
-## forward and backward short thunks, and long thunk reuse across remote
-## clusters. With the default 124MiB function-fragment cluster size, call
-## relaxation sees:
+## one-hop short thunks, and long thunk reuse across remote clusters. With the
+## default maximum thunk chain length and default 124MiB function-fragment
+## cluster size, call relaxation sees:
##
## A -> B direct same cluster
## B -> C direct forward
@@ -40,14 +40,15 @@
# RUN: --compact-code-model --relax-exp \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-bolt %t -o %t.hfe.bolt --data %t.fdata --split-functions \
-# RUN: --compact-code-model --relax-exp --hot-functions-at-end \
+# RUN: --compact-code-model --relax-exp \
+# RUN: --hot-functions-at-end \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D_0,__AArch64_backward_short_call_A_1,__AArch64_backward_long_call_A_0,__AArch64_backward_long_call_D_1 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_Thunk_D_0,__AArch64_backward_Thunk_A_1,__AArch64_backward_ADRPThunk_A_0,__AArch64_backward_ADRPThunk_D_1 \
# RUN: %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_short_call_D_0,__AArch64_forward_short_call_A_1,__AArch64_backward_short_call_A_2,__AArch64_forward_long_call_D_0 \
+# RUN: --disassemble-symbols=A,B,C,D,A.cold.0,B.cold.0,C.cold.0,D.cold.0,__AArch64_forward_Thunk_A_0,__AArch64_forward_Thunk_D_1,__AArch64_backward_Thunk_A_2,__AArch64_forward_ADRPThunk_D_0 \
# RUN: %t.hfe.bolt | FileCheck %s --check-prefix=CHECK-HFE-OUTPUT
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
@@ -63,12 +64,12 @@
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
-# CHECK-BOLT: BOLT-INFO: relaxed 2 short cluster calls with thunks
-# CHECK-BOLT: BOLT-INFO: relaxed 4 long cluster calls with thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 2 calls with short thunks
+# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with long thunks
# CHECK-BOLT: BOLT-INFO: 2 short thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks created
# CHECK-BOLT: BOLT-INFO: 2 long thunks reused
-# CHECK-BOLT: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT: BOLT-INFO: relaxed 8 unconditional branches
# CHECK-BOLT: BOLT-INFO: 12 branch thunks created
# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
@@ -84,12 +85,12 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 short cluster calls with thunks
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 2 long cluster calls with thunks
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 calls with short thunks
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 2 calls with long thunks
# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks created
# CHECK-BOLT-HFE: BOLT-INFO: 1 long thunks reused
-# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 cross-cluster branches
+# CHECK-BOLT-HFE: BOLT-INFO: relaxed 8 unconditional branches
# CHECK-BOLT-HFE: BOLT-INFO: 12 branch thunks created
# CHECK-SECTIONS: .text
@@ -175,64 +176,64 @@ D:
# CHECK-OUTPUT: <A>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_D_0>
# CHECK-OUTPUT: <B>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-OUTPUT: <__AArch64_forward_short_call_D_0>:
+# CHECK-OUTPUT: <__AArch64_forward_Thunk_D_0>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
-# CHECK-OUTPUT: <__AArch64_backward_short_call_A_1>:
+# CHECK-OUTPUT: <__AArch64_backward_Thunk_A_1>:
# CHECK-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-OUTPUT: <C>:
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-OUTPUT: <D>:
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A_1>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_Thunk_A_1>
# CHECK-OUTPUT: <A.cold.0>:
# CHECK-OUTPUT: <B.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_0>
# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <D>
-# CHECK-OUTPUT: <__AArch64_backward_long_call_A_0>:
+# CHECK-OUTPUT: <__AArch64_backward_ADRPThunk_A_0>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
-# CHECK-OUTPUT: <__AArch64_backward_long_call_D_1>:
+# CHECK-OUTPUT: <__AArch64_backward_ADRPThunk_D_1>:
# CHECK-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-OUTPUT-NEXT: {{.*}} br x16
# CHECK-OUTPUT: <C.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x3
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_0>
# CHECK-OUTPUT: <D.cold.0>:
# CHECK-OUTPUT-NEXT: {{.*}} mov x0, #0x4
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_A_0>
-# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_long_call_D_1>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_A_0>
+# CHECK-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_ADRPThunk_D_1>
# CHECK-HFE-OUTPUT: <A.cold.0>:
# CHECK-HFE-OUTPUT: <B.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x2
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_A_1>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D_0>
-
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_A_1>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_A_0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_ADRPThunk_D_0>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_long_call_D_0>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_ADRPThunk_D_0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} adrp x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} add x16, x16, {{.*}}
# CHECK-HFE-OUTPUT-NEXT: {{.*}} br x16
+# CHECK-HFE-OUTPUT: <__AArch64_forward_Thunk_A_0>:
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
+
# CHECK-HFE-OUTPUT: <C.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x3
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
@@ -240,23 +241,23 @@ D:
# CHECK-HFE-OUTPUT: <D.cold.0>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} mov x0, #0x4
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_long_call_D_0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_ADRPThunk_D_0>
# CHECK-HFE-OUTPUT: <A>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <B>
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_short_call_D_0>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_forward_Thunk_D_1>
# CHECK-HFE-OUTPUT: <B>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <C>
-# CHECK-HFE-OUTPUT: <__AArch64_forward_short_call_D_0>:
+# CHECK-HFE-OUTPUT: <__AArch64_forward_Thunk_D_1>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <D>
-# CHECK-HFE-OUTPUT: <__AArch64_backward_short_call_A_2>:
+# CHECK-HFE-OUTPUT: <__AArch64_backward_Thunk_A_2>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} b {{.*}} <A>
# CHECK-HFE-OUTPUT: <C>:
# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <A>
# CHECK-HFE-OUTPUT: <D>:
-# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_short_call_A_2>
+# CHECK-HFE-OUTPUT-NEXT: {{.*}} bl {{.*}} <__AArch64_backward_Thunk_A_2>
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
index dd3e287b7da41..eaae30479decc 100644
--- a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
+++ b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
@@ -16,7 +16,7 @@
# RUN: --skip-funcs='^bar$' \
# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_long_call_bar_0 \
+# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_ADRPThunk_bar_0 \
# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch \
@@ -27,17 +27,17 @@
# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
# RUN: | FileCheck %s --check-prefix=BRANCH-BOLT
# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=foo,bar,bar_body,.bar_ret,padding,__AArch64_forward_branch_chain_0 \
+# RUN: --disassemble-symbols=foo,bar,bar_body,.bar_ret,padding,__AArch64_forward_Thunk_0 \
# RUN: %t.branch.bolt | FileCheck %s --check-prefix=BRANCH
# TAILCALL-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
# TAILCALL-BOLT: BOLT-INFO: starting experimental relaxation pass
-# TAILCALL-BOLT: BOLT-INFO: relaxed 1 long cluster calls with thunks
+# TAILCALL-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
# TAILCALL-BOLT: BOLT-INFO: 1 long thunks created
# BRANCH-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
# BRANCH-BOLT: BOLT-INFO: starting experimental relaxation pass
-# BRANCH-BOLT: BOLT-INFO: relaxed 1 cross-cluster branches
+# BRANCH-BOLT: BOLT-INFO: relaxed 1 unconditional branches
# BRANCH-BOLT: BOLT-INFO: 1 branch thunks created
.text
@@ -94,9 +94,9 @@ padding:
#
# TAILCALL: <foo>:
# TAILCALL-NEXT: {{.*}} mov x16, #0x2a
-# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_long_call_bar_0>
+# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_ADRPThunk_bar_0>
#
-# TAILCALL: <__AArch64_forward_long_call_bar_0>:
+# TAILCALL: <__AArch64_forward_ADRPThunk_bar_0>:
# TAILCALL-NEXT: {{.*}} adrp x16,
# TAILCALL-NEXT: {{.*}} add x16, x16,
# TAILCALL-NEXT: {{.*}} br x16
@@ -116,9 +116,9 @@ padding:
#
# BRANCH: <foo>:
# BRANCH-NEXT: {{.*}} mov x16, #0x2a
-# BRANCH-NEXT: {{.*}} b 0x[[CHAIN:[0-9a-f]+]] <__AArch64_forward_branch_chain_0>
+# BRANCH-NEXT: {{.*}} b 0x[[CHAIN:[0-9a-f]+]] <__AArch64_forward_Thunk_0>
#
-# BRANCH: <__AArch64_forward_branch_chain_0>:
+# BRANCH: <__AArch64_forward_Thunk_0>:
# BRANCH-NEXT: [[CHAIN]]: {{.*}} b 0x[[BODY:[0-9a-f]+]] <bar+0x4>
#
# BRANCH: <padding>:
>From 02a8dd582894377c94bc3d8604d7d2b0935a9ba0 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sat, 12 Sep 2026 11:03:34 +0000
Subject: [PATCH 16/22] wrap clustered relaxation in a helper class and tidy up
/ add comments
---
bolt/include/bolt/Passes/LongJmp.h | 116 ---------
bolt/lib/Passes/LongJmp.cpp | 361 ++++++++++++++++++-----------
2 files changed, 225 insertions(+), 252 deletions(-)
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index 3a81fd54d5730..8c49f9a0bd4d6 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -10,7 +10,6 @@
#define BOLT_PASSES_LONGJMP_H
#include "bolt/Passes/BinaryPasses.h"
-#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
namespace llvm {
@@ -82,121 +81,6 @@ class LongJmpPass : public BinaryFunctionPass {
bool relaxLocalBranches(BinaryFunction &BF,
const BranchLivenessInfo *BLI = nullptr);
- /// A group of function fragments that are located within the longest direct
- /// branch/call instruction distance. Jumps within the cluster do not require
- /// a thunk. The cluster may span output sections and include thunks for jumps
- /// to targets outside. Backward thunks are inserted before the cluster, while
- /// forward thunks are inserted after it.
- struct FragmentCluster {
- /// Output code sections containing the first and last cluster fragments.
- SmallString<32> StartSectionName;
- SmallString<32> EndSectionName;
-
- /// Estimated size of the cluster in bytes.
- uint64_t Size{0};
-
- /// Estimated output offset of the cluster.
- uint64_t StartOffset{0};
-
- /// Number of function fragments in the cluster.
- size_t NumFragments{0};
-
- /// The indices of the first and last functions contributing fragments to
- /// this cluster. Used as insertion points for adding thunks to the output
- /// function list.
- size_t FirstFunctionIndex = -1;
- size_t LastFunctionIndex = -1;
-
- /// Thunks located after this cluster.
- BinaryFunctionListType ForwardThunkList;
-
- /// Thunks located before this cluster.
- BinaryFunctionListType BackwardThunkList;
-
- /// Long call thunks emitted by this cluster.
- ///
- /// <Function Symbol> -> <Thunk Function>.
- DenseMap<const MCSymbol *, BinaryFunction *> LongThunks;
-
- /// B-only thunks emitted by this cluster.
- ///
- /// <Destination Symbol> -> <Thunk Function>.
- DenseMap<const MCSymbol *, BinaryFunction *> BranchThunks;
-
- StringRef getThunkSectionName(bool IsForward) const {
- return IsForward ? EndSectionName : StartSectionName;
- }
-
- uint64_t getEndOffset() const { return StartOffset + Size; }
- };
-
- struct ClusterLayout {
- struct Position {
- unsigned Cluster;
- uint64_t Offset;
- };
-
- SmallVector<FragmentCluster, 4> Clusters;
- DenseMap<const BinaryBasicBlock *, Position> BBLayout;
- DenseMap<const MCSymbol *, Position> SymLayout;
- };
-
- struct ClusterReference {
- MCInst *Inst;
- const MCSymbol *TargetSymbol;
- uint64_t SourceOffset;
- uint64_t TargetOffset;
- unsigned SourceCluster;
- unsigned TargetCluster;
- };
-
- struct ClusterRelaxationWorklist {
- SmallVector<ClusterReference> OutOfLayoutCalls;
- SmallVector<SmallVector<ClusterReference>, 4> CallsByDistance;
- SmallVector<ClusterReference> Branches;
- };
-
- struct ClusterRelaxationStats {
- size_t NumShortThunkCalls = 0;
- size_t NumLongThunkCalls = 0;
- size_t NumShortThunks = 0;
- size_t NumShortThunksReused = 0;
- size_t NumLongThunks = 0;
- size_t NumLongThunksReused = 0;
- size_t NumBranchThunks = 0;
- size_t NumBranchThunksReused = 0;
- };
-
- ClusterLayout
- buildClusterLayout(BinaryContext &BC,
- const BinaryFunctionListType &OutputFunctions);
-
- /// Collect call and branch references that need cluster-level relaxation.
- ClusterRelaxationWorklist collectClusterRelaxationWorklist(
- BinaryContext &BC, const BinaryFunctionListType &OutputFunctions,
- const ClusterLayout &Layout);
-
- /// Return the first thunk symbol for an unconditional branch thunk chain.
- const MCSymbol *getOrCreateBranchThunkChain(BinaryContext &BC,
- ClusterLayout &Layout,
- ClusterRelaxationStats &Stats,
- const ClusterReference &Ref,
- unsigned MaxThunks);
-
- /// Relax calls using function fragment clusters.
- void relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
- ClusterRelaxationWorklist &Worklist,
- ClusterRelaxationStats &Stats);
-
- /// Relax direct unconditional branches using function fragment clusters.
- void relaxUnconditionalBranches(BinaryContext &BC, ClusterLayout &Layout,
- ClusterRelaxationWorklist &Worklist,
- ClusterRelaxationStats &Stats);
-
- /// Insert all thunks owned by the fragment cluster layout.
- void insertClusterThunks(BinaryFunctionListType &OutputFunctions,
- ClusterLayout &Layout);
-
/// Relax calls and direct unconditional branches using one cluster layout.
void relaxWithClusters(BinaryContext &BC);
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index b0b3cfbe59eeb..a0284684a4489 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -16,6 +16,7 @@
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/MathExtras.h"
@@ -76,17 +77,6 @@ static void relaxStubToLongJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
BC.MIB->applyBTIFixupToTarget(StubBB);
}
-static bool isWithinClusterRange(uint64_t SourceOffset, uint64_t TargetOffset) {
- const uint64_t Distance = SourceOffset <= TargetOffset
- ? TargetOffset - SourceOffset
- : SourceOffset - TargetOffset;
- return Distance < opts::MaxClusterSize;
-}
-
-static unsigned getClusterDistance(unsigned A, unsigned B) {
- return A > B ? A - B : B - A;
-}
-
static BinaryBasicBlock *getBBAtHotColdSplitPoint(BinaryFunction &Func) {
if (!Func.isSplit() || Func.empty())
return nullptr;
@@ -1043,8 +1033,131 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
return true;
}
-static uint64_t estimateFragmentSize(const BinaryFunction &BF,
- const FunctionFragment &FF) {
+namespace {
+class ClusteredRelaxation {
+public:
+ ClusteredRelaxation(BinaryContext &BC,
+ BinaryFunctionListType &OutputFunctions)
+ : BC(BC), OutputFunctions(OutputFunctions) {}
+
+ bool run();
+
+private:
+ /// A group of function fragments that are located within the longest direct
+ /// branch/call instruction distance. Jumps within the cluster do not require
+ /// a thunk. The cluster may span output sections and include thunks for jumps
+ /// to targets outside. Backward thunks are inserted before the cluster, while
+ /// forward thunks are inserted after it.
+ struct FragmentCluster {
+ /// Output code sections containing the first and last cluster fragments.
+ SmallString<32> StartSectionName;
+ SmallString<32> EndSectionName;
+
+ /// Estimated size of the cluster in bytes.
+ uint64_t Size{0};
+
+ /// Estimated output offset of the cluster.
+ uint64_t StartOffset{0};
+
+ /// Number of function fragments in the cluster.
+ size_t NumFragments{0};
+
+ /// The indices of the first and last functions contributing fragments to
+ /// this cluster. Used as insertion points for adding thunks to the output
+ /// function list.
+ size_t FirstFunctionIndex = -1;
+ size_t LastFunctionIndex = -1;
+
+ /// Thunks located after this cluster.
+ BinaryFunctionListType ForwardThunkList;
+
+ /// Thunks located before this cluster.
+ BinaryFunctionListType BackwardThunkList;
+
+ /// Long call thunks emitted by this cluster.
+ ///
+ /// <Target Symbol> -> <Thunk Function>.
+ DenseMap<const MCSymbol *, BinaryFunction *> LongThunks;
+
+ /// B-only thunks emitted by this cluster.
+ ///
+ /// <Target Symbol> -> <Thunk Function>.
+ DenseMap<const MCSymbol *, BinaryFunction *> BranchThunks;
+
+ StringRef getThunkSectionName(bool IsForward) const {
+ return IsForward ? EndSectionName : StartSectionName;
+ }
+
+ uint64_t getEndOffset() const { return StartOffset + Size; }
+ };
+
+ struct Position {
+ unsigned Cluster;
+ uint64_t Offset;
+ };
+
+ /// Relaxation info for out-of-range cross-cluster references.
+ struct OutOfRangeRef {
+ MCInst *Inst;
+ const MCSymbol *TargetSymbol;
+ uint64_t SourceOffset;
+ uint64_t TargetOffset;
+ unsigned SourceCluster;
+ unsigned TargetCluster;
+ };
+
+ static bool isWithinClusterRange(uint64_t SourceOffset,
+ uint64_t TargetOffset);
+ static unsigned getClusterDistance(unsigned A, unsigned B);
+ static uint64_t estimateFragmentSize(const BinaryFunction &BF,
+ const FunctionFragment &FF);
+ void buildLayout();
+ void collectOutOfRangeReferences();
+ const MCSymbol *getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
+ unsigned MaxThunks);
+ void relaxCalls();
+ void relaxUnconditionalBranches();
+ void insertThunks();
+
+ BinaryContext &BC;
+ BinaryFunctionListType &OutputFunctions;
+
+ /// Estimated cluster layout and source/target position maps.
+ SmallVector<FragmentCluster, 4> Clusters;
+ DenseMap<const BinaryBasicBlock *, Position> BBLayout;
+ DenseMap<const MCSymbol *, Position> SymLayout;
+
+ /// Out-of-range calls and branches grouped for relaxation.
+ SmallVector<OutOfRangeRef> OutOfLayoutCalls;
+ SmallVector<SmallVector<OutOfRangeRef>, 4> CallsByDistance;
+ SmallVector<OutOfRangeRef> Branches;
+
+ /// Relaxation counters reported in BOLT-INFO.
+ size_t NumShortThunkCalls = 0;
+ size_t NumLongThunkCalls = 0;
+ size_t NumShortThunks = 0;
+ size_t NumShortThunksReused = 0;
+ size_t NumLongThunks = 0;
+ size_t NumLongThunksReused = 0;
+ size_t NumBranchThunks = 0;
+ size_t NumBranchThunksReused = 0;
+};
+} // namespace
+
+bool ClusteredRelaxation::isWithinClusterRange(uint64_t SourceOffset,
+ uint64_t TargetOffset) {
+ const uint64_t Distance = SourceOffset <= TargetOffset
+ ? TargetOffset - SourceOffset
+ : SourceOffset - TargetOffset;
+ return Distance < opts::MaxClusterSize;
+}
+
+unsigned ClusteredRelaxation::getClusterDistance(unsigned A, unsigned B) {
+ return A > B ? A - B : B - A;
+}
+
+uint64_t ClusteredRelaxation::estimateFragmentSize(const BinaryFunction &BF,
+ const FunctionFragment &FF) {
uint64_t Size = 0;
for (const BinaryBasicBlock *BB : FF)
Size += BB->estimateSize();
@@ -1060,9 +1173,7 @@ static uint64_t estimateFragmentSize(const BinaryFunction &BF,
return Size;
}
-LongJmpPass::ClusterLayout
-LongJmpPass::buildClusterLayout(BinaryContext &BC,
- const BinaryFunctionListType &OutputFunctions) {
+void ClusteredRelaxation::buildLayout() {
struct OutputFragment {
const FunctionFragment *FF;
size_t FunctionIndex;
@@ -1075,7 +1186,6 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
size_t End;
};
- ClusterLayout Layout;
SmallVector<SmallVector<OutputFragment>, 4> FragmentsBySection;
StringMap<size_t> SectionToBucket;
auto addOrderedFragment = [&](OutputFragment &&Fragment) {
@@ -1116,11 +1226,13 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
for (const OutputFragment &Fragment : FragmentsBySection[SectionIndex])
OrderedFragments.push_back(&Fragment);
+ // Fragment clusters are built starting from hot code.
auto buildClusterRanges = [&]() {
SmallVector<FragmentRange> ClusterRanges;
if (OrderedFragments.empty())
return ClusterRanges;
+ // Hot fragments appear first, so perform forward walk.
if (!opts::HotFunctionsAtEnd) {
size_t Begin = 0;
uint64_t Size = OrderedFragments[0]->Size;
@@ -1136,6 +1248,7 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
return ClusterRanges;
}
+ // Hot fragments appear last, so perform reverse walk.
size_t End = OrderedFragments.size();
uint64_t Size = OrderedFragments.back()->Size;
for (size_t I = End - 1; I > 0;) {
@@ -1154,8 +1267,8 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
uint64_t LayoutOffset = 0;
auto addFragmentToCluster = [&](const OutputFragment &Fragment) {
- FragmentCluster &FC = Layout.Clusters.back();
- const unsigned ClusterNum = Layout.Clusters.size() - 1;
+ FragmentCluster &FC = Clusters.back();
+ const unsigned ClusterNum = Clusters.size() - 1;
BinaryFunction &BF = *OutputFunctions[Fragment.FunctionIndex];
const FunctionFragment &FF = *Fragment.FF;
const uint64_t FragmentOffset = LayoutOffset;
@@ -1173,18 +1286,18 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
// Map primary entry points.
if (FF.isMainFragment())
for (const MCSymbol *Symbol : BF.getSymbols())
- Layout.SymLayout[Symbol] = {ClusterNum, FragmentOffset};
+ SymLayout[Symbol] = {ClusterNum, FragmentOffset};
uint64_t BBOffset = FragmentOffset;
for (const BinaryBasicBlock *BB : FF) {
- Layout.BBLayout[BB] = {ClusterNum, BBOffset};
+ BBLayout[BB] = {ClusterNum, BBOffset};
// Map the local BB label.
if (const MCSymbol *Label = BB->getLabel())
- Layout.SymLayout[Label] = {ClusterNum, BBOffset};
+ SymLayout[Label] = {ClusterNum, BBOffset};
// Map the secondary entry point, which can differ from the BB label.
if (MCSymbol *Label = BF.getLabelAtOffset(BB->getOffset()))
if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(Label))
- Layout.SymLayout[EntrySymbol] = {ClusterNum, BBOffset};
+ SymLayout[EntrySymbol] = {ClusterNum, BBOffset};
BBOffset += BB->estimateSize();
}
@@ -1194,46 +1307,37 @@ LongJmpPass::buildClusterLayout(BinaryContext &BC,
};
for (const FragmentRange &Range : buildClusterRanges()) {
- Layout.Clusters.emplace_back();
+ Clusters.emplace_back();
for (size_t I = Range.Begin; I < Range.End; ++I)
addFragmentToCluster(*OrderedFragments[I]);
}
- if (Layout.Clusters.empty())
- return Layout;
+ if (Clusters.empty())
+ return;
LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << LayoutOffset
<< '\n');
// Print cluster stats.
- BC.outs() << "BOLT-INFO: built " << Layout.Clusters.size()
+ BC.outs() << "BOLT-INFO: built " << Clusters.size()
<< " function fragment cluster(s)\n";
- for (size_t I = 0; I < Layout.Clusters.size(); ++I) {
- const FragmentCluster &FC = Layout.Clusters[I];
+ for (size_t I = 0; I < Clusters.size(); ++I) {
+ const FragmentCluster &FC = Clusters[I];
BC.outs() << "BOLT-INFO: cluster: " << I << '\n'
<< "BOLT-INFO: " << FC.NumFragments << " fragment(s)\n"
<< "BOLT-INFO: " << FC.Size << " estimated bytes\n";
}
-
- return Layout;
}
-LongJmpPass::ClusterRelaxationWorklist
-LongJmpPass::collectClusterRelaxationWorklist(
- BinaryContext &BC, const BinaryFunctionListType &OutputFunctions,
- const ClusterLayout &Layout) {
- auto &Clusters = Layout.Clusters;
- auto &BBLayout = Layout.BBLayout;
- auto &SymLayout = Layout.SymLayout;
-
- ClusterRelaxationWorklist Worklist;
- Worklist.CallsByDistance.resize(Clusters.size());
+void ClusteredRelaxation::collectOutOfRangeReferences() {
+ CallsByDistance.resize(Clusters.size());
auto isPrimaryEntryTarget = [&](const MCSymbol *TargetSymbol) {
uint64_t EntryID = 0;
const BinaryFunction *BF = BC.getFunctionForSymbol(TargetSymbol, &EntryID);
return BF && EntryID == 0;
};
+ // Walk all instructions once and collect both branches and calls.
for (BinaryFunction *BF : OutputFunctions) {
if (!BC.shouldEmit(*BF) || BF->isPatch())
continue;
@@ -1242,7 +1346,7 @@ LongJmpPass::collectClusterRelaxationWorklist(
auto SourceIt = BBLayout.find(&BB);
if (SourceIt == BBLayout.end())
continue;
- const ClusterLayout::Position Source = SourceIt->second;
+ const Position Source = SourceIt->second;
uint64_t InstOffset = Source.Offset;
for (MCInst &Inst : BB) {
@@ -1263,11 +1367,16 @@ LongJmpPass::collectClusterRelaxationWorklist(
auto TargetIt = SymLayout.find(TargetSymbol);
const bool Found = TargetIt != SymLayout.end();
// Unmapped targets use maximum offset and are treated as out of range.
- const ClusterLayout::Position Target =
- Found ? TargetIt->second : ClusterLayout::Position{-1u, -1ULL};
- const ClusterReference Reference{&Inst, TargetSymbol,
- SourceOffset, Target.Offset,
- Source.Cluster, Target.Cluster};
+ const Position Target = Found ? TargetIt->second : Position{-1u, -1ULL};
+
+ // Skip already in-range references that do not need relaxation.
+ if (Source.Cluster == Target.Cluster ||
+ isWithinClusterRange(SourceOffset, Target.Offset))
+ continue;
+
+ const OutOfRangeRef Reference{&Inst, TargetSymbol,
+ SourceOffset, Target.Offset,
+ Source.Cluster, Target.Cluster};
const bool UseBranchChain =
Found && (IsUncondBranch ||
(IsTailCall && !isPrimaryEntryTarget(TargetSymbol)));
@@ -1277,41 +1386,28 @@ LongJmpPass::collectClusterRelaxationWorklist(
// thunks that may clobber x16/x17.
if (IsTailCall)
BC.MIB->convertTailCallToJmp(Inst);
- if (Source.Cluster == Target.Cluster)
- continue;
- if (isWithinClusterRange(SourceOffset, Target.Offset))
- continue;
- Worklist.Branches.push_back(Reference);
+ Branches.push_back(Reference);
continue;
}
assert(IsCall && "expected call after branch-chain handling");
- if (Target.Cluster == Source.Cluster)
- continue;
-
- if (isWithinClusterRange(SourceOffset, Target.Offset))
- continue;
-
if (Reference.TargetCluster == -1u) {
- Worklist.OutOfLayoutCalls.push_back(Reference);
+ OutOfLayoutCalls.push_back(Reference);
} else {
const unsigned Distance = getClusterDistance(Reference.SourceCluster,
Reference.TargetCluster);
- Worklist.CallsByDistance[Distance].push_back(Reference);
+ CallsByDistance[Distance].push_back(Reference);
}
}
}
}
-
- return Worklist;
}
-const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
- BinaryContext &BC, ClusterLayout &Layout, ClusterRelaxationStats &Stats,
- const ClusterReference &Ref, unsigned MaxThunks) {
- auto &Clusters = Layout.Clusters;
+const MCSymbol *
+ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
+ unsigned MaxThunks) {
const unsigned SourceCluster = Ref.SourceCluster;
const unsigned TargetCluster = Ref.TargetCluster;
const bool IsForward = SourceCluster < TargetCluster;
@@ -1319,8 +1415,7 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
const unsigned NumHops = getClusterDistance(SourceCluster, TargetCluster);
auto createBranchThunk = [&](const MCSymbol *TargetSymbol) {
- const size_t ThunkNumber =
- IsCall ? Stats.NumShortThunks++ : Stats.NumBranchThunks++;
+ const size_t ThunkNumber = IsCall ? NumShortThunks++ : NumBranchThunks++;
std::string ThunkName =
(Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
"Thunk_" + (IsCall ? Twine(Ref.TargetSymbol->getName()) + "_" : "") +
@@ -1335,15 +1430,17 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
};
auto registerBranchThunk = [&](FragmentCluster &Cluster,
+ const MCSymbol *TargetSymbol,
BinaryFunction *Thunk) {
uint64_t EntryID = 0;
const BinaryFunction *BF =
- IsCall ? BC.getFunctionForSymbol(Ref.TargetSymbol, &EntryID) : nullptr;
+ IsCall ? BC.getFunctionForSymbol(TargetSymbol, &EntryID) : nullptr;
+ // Call-style branch thunks may be reused through primary-entry aliases.
if (BF && EntryID == 0)
for (const MCSymbol *Symbol : BF->getSymbols())
Cluster.BranchThunks[Symbol] = Thunk;
else
- Cluster.BranchThunks[Ref.TargetSymbol] = Thunk;
+ Cluster.BranchThunks[TargetSymbol] = Thunk;
};
auto getOrCreateBranchThunk = [&](FragmentCluster &Cluster,
@@ -1351,9 +1448,9 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
auto It = Cluster.BranchThunks.find(Ref.TargetSymbol);
if (It != Cluster.BranchThunks.end()) {
if (IsCall)
- ++Stats.NumShortThunksReused;
+ ++NumShortThunksReused;
else
- ++Stats.NumBranchThunksReused;
+ ++NumBranchThunksReused;
return It->second;
}
@@ -1362,7 +1459,7 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
auto &ThunkList =
IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
ThunkList.push_back(Thunk);
- registerBranchThunk(Cluster, Thunk);
+ registerBranchThunk(Cluster, Ref.TargetSymbol, Thunk);
return Thunk;
};
@@ -1379,6 +1476,7 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
uint64_t CurrentOffset = Ref.SourceOffset;
unsigned NextHop = 0;
while (!isWithinClusterRange(CurrentOffset, Ref.TargetOffset)) {
+ // Stop if the chain would exceed the configured thunk budget.
if (ThunkClusters.size() == MaxThunks)
return nullptr;
@@ -1401,6 +1499,7 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
BinaryFunction *FirstThunk = nullptr;
const MCSymbol *NextTarget = Ref.TargetSymbol;
+ // Reverse walk since each thunk depends on its target.
for (const unsigned Cluster : llvm::reverse(ThunkClusters)) {
FirstThunk = getOrCreateBranchThunk(Clusters[Cluster], NextTarget);
NextTarget = FirstThunk->getSymbol();
@@ -1409,16 +1508,10 @@ const MCSymbol *LongJmpPass::getOrCreateBranchThunkChain(
return FirstThunk ? FirstThunk->getSymbol() : Ref.TargetSymbol;
}
-void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
- ClusterRelaxationWorklist &Worklist,
- ClusterRelaxationStats &Stats) {
- auto &Clusters = Layout.Clusters;
- auto &OutOfLayoutCalls = Worklist.OutOfLayoutCalls;
- auto &CallsByDistance = Worklist.CallsByDistance;
-
+void ClusteredRelaxation::relaxCalls() {
auto createLongThunk = [&](const MCSymbol *TargetSymbol, bool IsForward) {
BinaryFunction *Thunk = nullptr;
- const size_t ThunkNumber = Stats.NumLongThunks++;
+ const size_t ThunkNumber = NumLongThunks++;
std::string ThunkName =
(Twine("__AArch64_") + (IsForward ? "forward_" : "backward_") +
"ADRPThunk_" + TargetSymbol->getName() + "_" + Twine(ThunkNumber))
@@ -1435,6 +1528,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
uint64_t EntryID = 0;
const BinaryFunction *BF = BC.getFunctionForSymbol(Callee, &EntryID);
const bool IsPrimaryEntry = EntryID == 0;
+ // Register thunks for all symbols associated with the function.
if (BF && IsPrimaryEntry)
for (const MCSymbol *Symbol : BF->getSymbols())
FC.LongThunks[Symbol] = Thunk;
@@ -1442,13 +1536,13 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
FC.LongThunks[Callee] = Thunk;
};
- auto getOrCreateLongThunk = [&](const ClusterReference &Call) {
+ auto getOrCreateLongThunk = [&](const OutOfRangeRef &Call) {
const MCSymbol *TargetSymbol = Call.TargetSymbol;
const unsigned SourceCluster = Call.SourceCluster;
const bool IsForward = SourceCluster < Call.TargetCluster;
FragmentCluster &FC = Clusters[SourceCluster];
if (auto It = FC.LongThunks.find(TargetSymbol); It != FC.LongThunks.end()) {
- ++Stats.NumLongThunksReused;
+ ++NumLongThunksReused;
return It->second;
}
@@ -1463,7 +1557,7 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
auto It = ReuseCluster->LongThunks.find(TargetSymbol);
if (It != ReuseCluster->LongThunks.end()) {
BinaryFunction *Thunk = It->second;
- ++Stats.NumLongThunksReused;
+ ++NumLongThunksReused;
return Thunk;
}
}
@@ -1474,71 +1568,63 @@ void LongJmpPass::relaxCalls(BinaryContext &BC, ClusterLayout &Layout,
BinaryFunctionListType &ThunkList =
IsForward ? FC.ForwardThunkList : FC.BackwardThunkList;
ThunkList.push_back(Thunk);
-
- // Register thunks for all symbols associated with the function.
registerLongThunk(FC, TargetSymbol, Thunk);
return Thunk;
};
- auto relaxCallWithLongThunk = [&](ClusterReference &Call) {
+ auto relaxCallWithLongThunk = [&](OutOfRangeRef &Call) {
BinaryFunction *Thunk = getOrCreateLongThunk(Call);
BC.MIB->replaceBranchTarget(*Call.Inst, Thunk->getSymbol(), BC.Ctx.get());
};
// Process out-of-layout targets first.
- for (ClusterReference &Call : OutOfLayoutCalls)
+ for (OutOfRangeRef &Call : OutOfLayoutCalls)
relaxCallWithLongThunk(Call);
- Stats.NumLongThunkCalls += OutOfLayoutCalls.size();
+ NumLongThunkCalls += OutOfLayoutCalls.size();
// Process known targets from farthest to nearest for maximizing thunk reuse.
for (auto &Calls : llvm::reverse(CallsByDistance)) {
- for (ClusterReference &Call : Calls) {
+ for (OutOfRangeRef &Call : Calls) {
// Attempt a branch thunk chain if length is below threshold.
- if (const MCSymbol *Target = getOrCreateBranchThunkChain(
- BC, Layout, Stats, Call, opts::MaxThunkChainLength)) {
+ if (const MCSymbol *Target =
+ getOrCreateBranchThunkChain(Call, opts::MaxThunkChainLength)) {
BC.MIB->replaceBranchTarget(*Call.Inst, Target, BC.Ctx.get());
- ++Stats.NumShortThunkCalls;
+ ++NumShortThunkCalls;
continue;
}
// Otherwise fall back to a long thunk.
- ++Stats.NumLongThunkCalls;
+ ++NumLongThunkCalls;
relaxCallWithLongThunk(Call);
}
}
- if (Stats.NumShortThunkCalls)
- BC.outs() << "BOLT-INFO: relaxed " << Stats.NumShortThunkCalls
+ if (NumShortThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << NumShortThunkCalls
<< " calls with short thunks\n";
- if (Stats.NumLongThunkCalls)
- BC.outs() << "BOLT-INFO: relaxed " << Stats.NumLongThunkCalls
+ if (NumLongThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << NumLongThunkCalls
<< " calls with long thunks\n";
- if (Stats.NumShortThunks)
- BC.outs() << "BOLT-INFO: " << Stats.NumShortThunks
- << " short thunks created\n";
+ if (NumShortThunks)
+ BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
- if (Stats.NumLongThunks)
- BC.outs() << "BOLT-INFO: " << Stats.NumLongThunks
- << " long thunks created\n";
+ if (NumLongThunks)
+ BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
- if (Stats.NumShortThunksReused)
- BC.outs() << "BOLT-INFO: " << Stats.NumShortThunksReused
+ if (NumShortThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumShortThunksReused
<< " short thunks reused\n";
- if (Stats.NumLongThunksReused)
- BC.outs() << "BOLT-INFO: " << Stats.NumLongThunksReused
+ if (NumLongThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumLongThunksReused
<< " long thunks reused\n";
}
-void LongJmpPass::relaxUnconditionalBranches(
- BinaryContext &BC, ClusterLayout &Layout,
- ClusterRelaxationWorklist &Worklist, ClusterRelaxationStats &Stats) {
- auto &Branches = Worklist.Branches;
-
- for (const ClusterReference &Branch : Branches) {
- const MCSymbol *Target = getOrCreateBranchThunkChain(
- BC, Layout, Stats, Branch, /*MaxThunks=*/-1u);
+void ClusteredRelaxation::relaxUnconditionalBranches() {
+ for (const OutOfRangeRef &Branch : Branches) {
+ const MCSymbol *Target =
+ getOrCreateBranchThunkChain(Branch, /*MaxThunks=*/-1u);
assert(Target && "expected branch thunk chain");
BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
}
@@ -1547,17 +1633,15 @@ void LongJmpPass::relaxUnconditionalBranches(
BC.outs() << "BOLT-INFO: relaxed " << Branches.size()
<< " unconditional branches\n";
- if (Stats.NumBranchThunks)
- BC.outs() << "BOLT-INFO: " << Stats.NumBranchThunks
- << " branch thunks created\n";
+ if (NumBranchThunks)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
- if (Stats.NumBranchThunksReused)
- BC.outs() << "BOLT-INFO: " << Stats.NumBranchThunksReused
+ if (NumBranchThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunksReused
<< " branch thunks reused\n";
}
-void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
- ClusterLayout &Layout) {
+void ClusteredRelaxation::insertThunks() {
struct ThunkInsertion {
size_t Position;
bool IsForward;
@@ -1565,7 +1649,7 @@ void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
};
SmallVector<ThunkInsertion> Insertions;
- for (FragmentCluster &Cluster : Layout.Clusters) {
+ for (FragmentCluster &Cluster : Clusters) {
if (!Cluster.BackwardThunkList.empty())
Insertions.push_back({Cluster.FirstFunctionIndex, /*IsForward=*/false,
&Cluster.BackwardThunkList});
@@ -1591,23 +1675,28 @@ void LongJmpPass::insertClusterThunks(BinaryFunctionListType &OutputFunctions,
Insertion.ThunkList->begin(), Insertion.ThunkList->end());
}
-void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
- BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
- ClusterLayout Layout = buildClusterLayout(BC, OutputFunctions);
+bool ClusteredRelaxation::run() {
+ buildLayout();
+ if (Clusters.empty())
+ return false;
- if (Layout.Clusters.empty())
- return;
-
- ClusterRelaxationWorklist Worklist =
- collectClusterRelaxationWorklist(BC, OutputFunctions, Layout);
- ClusterRelaxationStats Stats;
- relaxCalls(BC, Layout, Worklist, Stats);
- relaxUnconditionalBranches(BC, Layout, Worklist, Stats);
- insertClusterThunks(OutputFunctions, Layout);
+ collectOutOfRangeReferences();
+ relaxCalls();
+ relaxUnconditionalBranches();
+ insertThunks();
LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
for (const auto *BF : OutputFunctions) { dbgs() << *BF << '\n'; });
+ return true;
+}
+
+void LongJmpPass::relaxWithClusters(BinaryContext &BC) {
+ BinaryFunctionListType OutputFunctions = BC.getOutputBinaryFunctions();
+ ClusteredRelaxation Relaxation(BC, OutputFunctions);
+ if (!Relaxation.run())
+ return;
+
BC.updateOutputBinaryFunctions(std::move(OutputFunctions));
}
>From 1aece6ef30d7b17f58c4b466b75e25bb3ddbd8c3 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Sun, 20 Sep 2026 08:55:49 +0000
Subject: [PATCH 17/22] account for estimated thunk bytes when range checking
---
bolt/lib/Passes/LongJmp.cpp | 198 +++++++++++++-----
.../AArch64/relax-branches-with-thunk-chain.s | 24 ++-
.../test/AArch64/relax-cross-fragment-calls.s | 32 ++-
3 files changed, 187 insertions(+), 67 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index a0284684a4489..dc7713513db9d 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -15,6 +15,7 @@
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
@@ -1043,6 +1044,9 @@ class ClusteredRelaxation {
bool run();
private:
+ static constexpr uint64_t ShortThunkSize = 4;
+ static constexpr uint64_t LongThunkSize = 12;
+
/// A group of function fragments that are located within the longest direct
/// branch/call instruction distance. Jumps within the cluster do not require
/// a thunk. The cluster may span output sections and include thunks for jumps
@@ -1059,6 +1063,12 @@ class ClusteredRelaxation {
/// Estimated output offset of the cluster.
uint64_t StartOffset{0};
+ /// Estimated bytes for thunks that may be inserted around this cluster.
+ uint64_t EstimatedThunkBytes{0};
+
+ /// Actual bytes for thunks emitted by this cluster.
+ uint64_t ActualThunkBytes{0};
+
/// Number of function fragments in the cluster.
size_t NumFragments{0};
@@ -1106,13 +1116,13 @@ class ClusteredRelaxation {
unsigned TargetCluster;
};
- static bool isWithinClusterRange(uint64_t SourceOffset,
- uint64_t TargetOffset);
static unsigned getClusterDistance(unsigned A, unsigned B);
static uint64_t estimateFragmentSize(const BinaryFunction &BF,
const FunctionFragment &FF);
void buildLayout();
+ void printStats() const;
void collectOutOfRangeReferences();
+ void estimateThunkBytes();
const MCSymbol *getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
unsigned MaxThunks);
void relaxCalls();
@@ -1139,19 +1149,12 @@ class ClusteredRelaxation {
size_t NumShortThunksReused = 0;
size_t NumLongThunks = 0;
size_t NumLongThunksReused = 0;
+ size_t NumRelaxedBranches = 0;
size_t NumBranchThunks = 0;
size_t NumBranchThunksReused = 0;
};
} // namespace
-bool ClusteredRelaxation::isWithinClusterRange(uint64_t SourceOffset,
- uint64_t TargetOffset) {
- const uint64_t Distance = SourceOffset <= TargetOffset
- ? TargetOffset - SourceOffset
- : SourceOffset - TargetOffset;
- return Distance < opts::MaxClusterSize;
-}
-
unsigned ClusteredRelaxation::getClusterDistance(unsigned A, unsigned B) {
return A > B ? A - B : B - A;
}
@@ -1317,16 +1320,55 @@ void ClusteredRelaxation::buildLayout() {
LLVM_DEBUG(dbgs() << "LongJmp: estimated code size : " << LayoutOffset
<< '\n');
+}
- // Print cluster stats.
+void ClusteredRelaxation::printStats() const {
BC.outs() << "BOLT-INFO: built " << Clusters.size()
<< " function fragment cluster(s)\n";
for (size_t I = 0; I < Clusters.size(); ++I) {
const FragmentCluster &FC = Clusters[I];
BC.outs() << "BOLT-INFO: cluster: " << I << '\n'
<< "BOLT-INFO: " << FC.NumFragments << " fragment(s)\n"
- << "BOLT-INFO: " << FC.Size << " estimated bytes\n";
+ << "BOLT-INFO: " << FC.Size
+ << " estimated bytes without thunks\n"
+ << "BOLT-INFO: " << FC.EstimatedThunkBytes
+ << " estimated thunk bytes\n"
+ << "BOLT-INFO: " << FC.ActualThunkBytes
+ << " actual thunk bytes\n";
}
+
+ if (NumShortThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << NumShortThunkCalls
+ << " calls with short thunks\n";
+
+ if (NumLongThunkCalls)
+ BC.outs() << "BOLT-INFO: relaxed " << NumLongThunkCalls
+ << " calls with long thunks\n";
+
+ if (NumShortThunks)
+ BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
+
+ if (NumLongThunks)
+ BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
+
+ if (NumShortThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumShortThunksReused
+ << " short thunks reused\n";
+
+ if (NumLongThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumLongThunksReused
+ << " long thunks reused\n";
+
+ if (NumRelaxedBranches)
+ BC.outs() << "BOLT-INFO: relaxed " << NumRelaxedBranches
+ << " unconditional branches\n";
+
+ if (NumBranchThunks)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
+
+ if (NumBranchThunksReused)
+ BC.outs() << "BOLT-INFO: " << NumBranchThunksReused
+ << " branch thunks reused\n";
}
void ClusteredRelaxation::collectOutOfRangeReferences() {
@@ -1369,9 +1411,8 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
// Unmapped targets use maximum offset and are treated as out of range.
const Position Target = Found ? TargetIt->second : Position{-1u, -1ULL};
- // Skip already in-range references that do not need relaxation.
- if (Source.Cluster == Target.Cluster ||
- isWithinClusterRange(SourceOffset, Target.Offset))
+ // References within the same cluster do not need relaxation.
+ if (Source.Cluster == Target.Cluster)
continue;
const OutOfRangeRef Reference{&Inst, TargetSymbol,
@@ -1393,7 +1434,7 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
assert(IsCall && "expected call after branch-chain handling");
- if (Reference.TargetCluster == -1u) {
+ if (!Found) {
OutOfLayoutCalls.push_back(Reference);
} else {
const unsigned Distance = getClusterDistance(Reference.SourceCluster,
@@ -1405,6 +1446,68 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
}
}
+void ClusteredRelaxation::estimateThunkBytes() {
+ // Estimate in the same order as relaxation so reuse approximates the
+ // thunks that will actually be created below. This models the worst case
+ // by assuming selected thunks are necessary without performing range checks.
+ SmallVector<DenseSet<const MCSymbol *>, 4> BranchThunkTargets;
+ SmallVector<DenseSet<const MCSymbol *>, 4> LongThunkTargets;
+ BranchThunkTargets.resize(Clusters.size());
+ LongThunkTargets.resize(Clusters.size());
+
+ auto estimateBranchThunks = [&](const OutOfRangeRef &Ref) {
+ const bool IsForward = Ref.SourceCluster < Ref.TargetCluster;
+ auto getClusterAtHop = [&](unsigned Hop) {
+ return IsForward ? Ref.SourceCluster + Hop : Ref.SourceCluster - Hop;
+ };
+
+ const unsigned NumHops =
+ getClusterDistance(Ref.SourceCluster, Ref.TargetCluster);
+ for (unsigned Hop = 0; Hop < NumHops; ++Hop) {
+ const unsigned Cluster = getClusterAtHop(Hop);
+ if (BranchThunkTargets[Cluster].insert(Ref.TargetSymbol).second)
+ Clusters[Cluster].EstimatedThunkBytes += ShortThunkSize;
+ }
+ };
+
+ auto estimateLongThunk = [&](const OutOfRangeRef &Ref) {
+ const unsigned SourceCluster = Ref.SourceCluster;
+ const bool IsForward = SourceCluster < Ref.TargetCluster;
+ if (LongThunkTargets[SourceCluster].contains(Ref.TargetSymbol))
+ return;
+
+ unsigned ReuseCluster = -1u;
+ if (IsForward && SourceCluster > 0)
+ ReuseCluster = SourceCluster - 1;
+ else if (!IsForward && SourceCluster + 1 < Clusters.size())
+ ReuseCluster = SourceCluster + 1;
+
+ if (ReuseCluster != -1u &&
+ LongThunkTargets[ReuseCluster].contains(Ref.TargetSymbol))
+ return;
+
+ LongThunkTargets[SourceCluster].insert(Ref.TargetSymbol);
+ Clusters[SourceCluster].EstimatedThunkBytes += LongThunkSize;
+ };
+
+ for (const OutOfRangeRef &Call : OutOfLayoutCalls)
+ estimateLongThunk(Call);
+
+ for (auto &Calls : llvm::reverse(CallsByDistance)) {
+ for (const OutOfRangeRef &Call : Calls) {
+ const unsigned Distance =
+ getClusterDistance(Call.SourceCluster, Call.TargetCluster);
+ if (Distance <= opts::MaxThunkChainLength)
+ estimateBranchThunks(Call);
+ else
+ estimateLongThunk(Call);
+ }
+ }
+
+ for (const OutOfRangeRef &Branch : Branches)
+ estimateBranchThunks(Branch);
+}
+
const MCSymbol *
ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
unsigned MaxThunks) {
@@ -1460,6 +1563,7 @@ ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
IsForward ? Cluster.ForwardThunkList : Cluster.BackwardThunkList;
ThunkList.push_back(Thunk);
registerBranchThunk(Cluster, Ref.TargetSymbol, Thunk);
+ Cluster.ActualThunkBytes += ShortThunkSize;
return Thunk;
};
@@ -1472,10 +1576,21 @@ ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
return IsForward ? FC.getEndOffset() : FC.StartOffset;
};
+ auto isWithinClusterRange = [&](Position Source, Position Target) {
+ uint64_t EstimatedThunkBytes = 0;
+ for (unsigned I = Source.Cluster; I <= Target.Cluster; ++I)
+ EstimatedThunkBytes += Clusters[getClusterAtHop(I)].EstimatedThunkBytes;
+
+ const uint64_t Distance = Source.Offset <= Target.Offset
+ ? Target.Offset - Source.Offset
+ : Source.Offset - Target.Offset;
+ return Distance + EstimatedThunkBytes < opts::MaxClusterSize;
+ };
+
SmallVector<unsigned> ThunkClusters;
- uint64_t CurrentOffset = Ref.SourceOffset;
+ Position Current{0, Ref.SourceOffset};
unsigned NextHop = 0;
- while (!isWithinClusterRange(CurrentOffset, Ref.TargetOffset)) {
+ while (!isWithinClusterRange(Current, {NumHops, Ref.TargetOffset})) {
// Stop if the chain would exceed the configured thunk budget.
if (ThunkClusters.size() == MaxThunks)
return nullptr;
@@ -1483,7 +1598,7 @@ ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
unsigned BestHop = -1u;
for (unsigned Hop = NextHop; Hop < NumHops; ++Hop) {
const unsigned Cluster = getClusterAtHop(Hop);
- if (!isWithinClusterRange(CurrentOffset, getThunkOffset(Cluster)))
+ if (!isWithinClusterRange(Current, {Hop, getThunkOffset(Cluster)}))
break;
BestHop = Hop;
}
@@ -1493,8 +1608,8 @@ ClusteredRelaxation::getOrCreateBranchThunkChain(const OutOfRangeRef &Ref,
const unsigned BestCluster = getClusterAtHop(BestHop);
ThunkClusters.push_back(BestCluster);
- CurrentOffset = getThunkOffset(BestCluster);
NextHop = BestHop + 1;
+ Current = {BestHop, getThunkOffset(BestCluster)};
}
BinaryFunction *FirstThunk = nullptr;
@@ -1569,6 +1684,7 @@ void ClusteredRelaxation::relaxCalls() {
IsForward ? FC.ForwardThunkList : FC.BackwardThunkList;
ThunkList.push_back(Thunk);
registerLongThunk(FC, TargetSymbol, Thunk);
+ FC.ActualThunkBytes += LongThunkSize;
return Thunk;
};
@@ -1588,6 +1704,9 @@ void ClusteredRelaxation::relaxCalls() {
// Attempt a branch thunk chain if length is below threshold.
if (const MCSymbol *Target =
getOrCreateBranchThunkChain(Call, opts::MaxThunkChainLength)) {
+ // Skip already in-range calls that do not need relaxation.
+ if (Target == Call.TargetSymbol)
+ continue;
BC.MIB->replaceBranchTarget(*Call.Inst, Target, BC.Ctx.get());
++NumShortThunkCalls;
continue;
@@ -1597,28 +1716,6 @@ void ClusteredRelaxation::relaxCalls() {
relaxCallWithLongThunk(Call);
}
}
-
- if (NumShortThunkCalls)
- BC.outs() << "BOLT-INFO: relaxed " << NumShortThunkCalls
- << " calls with short thunks\n";
-
- if (NumLongThunkCalls)
- BC.outs() << "BOLT-INFO: relaxed " << NumLongThunkCalls
- << " calls with long thunks\n";
-
- if (NumShortThunks)
- BC.outs() << "BOLT-INFO: " << NumShortThunks << " short thunks created\n";
-
- if (NumLongThunks)
- BC.outs() << "BOLT-INFO: " << NumLongThunks << " long thunks created\n";
-
- if (NumShortThunksReused)
- BC.outs() << "BOLT-INFO: " << NumShortThunksReused
- << " short thunks reused\n";
-
- if (NumLongThunksReused)
- BC.outs() << "BOLT-INFO: " << NumLongThunksReused
- << " long thunks reused\n";
}
void ClusteredRelaxation::relaxUnconditionalBranches() {
@@ -1626,19 +1723,12 @@ void ClusteredRelaxation::relaxUnconditionalBranches() {
const MCSymbol *Target =
getOrCreateBranchThunkChain(Branch, /*MaxThunks=*/-1u);
assert(Target && "expected branch thunk chain");
+ // Skip already in-range branches that do not need relaxation.
+ if (Target == Branch.TargetSymbol)
+ continue;
BC.MIB->replaceBranchTarget(*Branch.Inst, Target, BC.Ctx.get());
+ ++NumRelaxedBranches;
}
-
- if (!Branches.empty())
- BC.outs() << "BOLT-INFO: relaxed " << Branches.size()
- << " unconditional branches\n";
-
- if (NumBranchThunks)
- BC.outs() << "BOLT-INFO: " << NumBranchThunks << " branch thunks created\n";
-
- if (NumBranchThunksReused)
- BC.outs() << "BOLT-INFO: " << NumBranchThunksReused
- << " branch thunks reused\n";
}
void ClusteredRelaxation::insertThunks() {
@@ -1681,8 +1771,10 @@ bool ClusteredRelaxation::run() {
return false;
collectOutOfRangeReferences();
+ estimateThunkBytes();
relaxCalls();
relaxUnconditionalBranches();
+ printStats();
insertThunks();
LLVM_DEBUG(dbgs() << "\nFunction layout with thunks:\n";
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index e8699ffe33d20..d4c0d5c696630 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -39,13 +39,19 @@
# CHECK-BOLT: BOLT-INFO: built 3 function fragment cluster(s)
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440604 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440604 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 8 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 8 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 117440584 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 117440584 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 24 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 16 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 67108944 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 16 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 8 actual thunk bytes
# CHECK-BOLT: BOLT-INFO: relaxed 7 unconditional branches
# CHECK-BOLT: BOLT-INFO: 8 branch thunks created
# CHECK-BOLT: BOLT-INFO: 2 branch thunks reused
@@ -53,13 +59,19 @@
# CHECK-BOLT-HFE: BOLT-INFO: built 3 function fragment cluster(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 67108944 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 67108944 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 16 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 8 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440604 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440604 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 24 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 12 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 117440584 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 8 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 4 actual thunk bytes
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 4 unconditional branches
# CHECK-BOLT-HFE: BOLT-INFO: 6 branch thunks created
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 21075d979dcd5..dfc4a24df3727 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -54,16 +54,24 @@
# CHECK-BOLT: BOLT-INFO: built 4 function fragment cluster(s)
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 88080456 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080456 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 16 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 12 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 24 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 88080440 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080440 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 16 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
+# CHECK-BOLT-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 28 actual thunk bytes
# CHECK-BOLT: BOLT-INFO: relaxed 2 calls with short thunks
# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with long thunks
# CHECK-BOLT: BOLT-INFO: 2 short thunks created
@@ -75,16 +83,24 @@
# CHECK-BOLT-HFE: BOLT-INFO: built 4 function fragment cluster(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080440 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080440 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 24 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 20 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080456 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080456 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 20 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 12 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 8 actual thunk bytes
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 calls with short thunks
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 2 calls with long thunks
# CHECK-BOLT-HFE: BOLT-INFO: 3 short thunks created
>From cea305b48db4c04e73b43174d962775a3d2bcf9c Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 22 Sep 2026 12:42:47 +0000
Subject: [PATCH 18/22] Conservatively estimate both short and long thunk sizes
---
bolt/lib/Passes/LongJmp.cpp | 40 ++++++-------------
.../AArch64/relax-call-short-thunk-reuse.s | 2 +-
.../test/AArch64/relax-cross-fragment-calls.s | 16 ++++----
3 files changed, 21 insertions(+), 37 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index dc7713513db9d..679e7a8c01a87 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1325,8 +1325,13 @@ void ClusteredRelaxation::buildLayout() {
void ClusteredRelaxation::printStats() const {
BC.outs() << "BOLT-INFO: built " << Clusters.size()
<< " function fragment cluster(s)\n";
+
for (size_t I = 0; I < Clusters.size(); ++I) {
const FragmentCluster &FC = Clusters[I];
+
+ assert(FC.ActualThunkBytes <= FC.EstimatedThunkBytes &&
+ "thunk estimate exceeded; range checks used a stale safety margin");
+
BC.outs() << "BOLT-INFO: cluster: " << I << '\n'
<< "BOLT-INFO: " << FC.NumFragments << " fragment(s)\n"
<< "BOLT-INFO: " << FC.Size
@@ -1447,9 +1452,8 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
}
void ClusteredRelaxation::estimateThunkBytes() {
- // Estimate in the same order as relaxation so reuse approximates the
- // thunks that will actually be created below. This models the worst case
- // by assuming selected thunks are necessary without performing range checks.
+ // Conservatively estimate both branch chains and long thunks without range
+ // checks or cross-cluster reuse. Deduplicate targets within each cluster.
SmallVector<DenseSet<const MCSymbol *>, 4> BranchThunkTargets;
SmallVector<DenseSet<const MCSymbol *>, 4> LongThunkTargets;
BranchThunkTargets.resize(Clusters.size());
@@ -1460,7 +1464,6 @@ void ClusteredRelaxation::estimateThunkBytes() {
auto getClusterAtHop = [&](unsigned Hop) {
return IsForward ? Ref.SourceCluster + Hop : Ref.SourceCluster - Hop;
};
-
const unsigned NumHops =
getClusterDistance(Ref.SourceCluster, Ref.TargetCluster);
for (unsigned Hop = 0; Hop < NumHops; ++Hop) {
@@ -1471,36 +1474,17 @@ void ClusteredRelaxation::estimateThunkBytes() {
};
auto estimateLongThunk = [&](const OutOfRangeRef &Ref) {
- const unsigned SourceCluster = Ref.SourceCluster;
- const bool IsForward = SourceCluster < Ref.TargetCluster;
- if (LongThunkTargets[SourceCluster].contains(Ref.TargetSymbol))
- return;
-
- unsigned ReuseCluster = -1u;
- if (IsForward && SourceCluster > 0)
- ReuseCluster = SourceCluster - 1;
- else if (!IsForward && SourceCluster + 1 < Clusters.size())
- ReuseCluster = SourceCluster + 1;
-
- if (ReuseCluster != -1u &&
- LongThunkTargets[ReuseCluster].contains(Ref.TargetSymbol))
- return;
-
- LongThunkTargets[SourceCluster].insert(Ref.TargetSymbol);
- Clusters[SourceCluster].EstimatedThunkBytes += LongThunkSize;
+ if (LongThunkTargets[Ref.SourceCluster].insert(Ref.TargetSymbol).second)
+ Clusters[Ref.SourceCluster].EstimatedThunkBytes += LongThunkSize;
};
for (const OutOfRangeRef &Call : OutOfLayoutCalls)
estimateLongThunk(Call);
- for (auto &Calls : llvm::reverse(CallsByDistance)) {
+ for (const auto &Calls : CallsByDistance) {
for (const OutOfRangeRef &Call : Calls) {
- const unsigned Distance =
- getClusterDistance(Call.SourceCluster, Call.TargetCluster);
- if (Distance <= opts::MaxThunkChainLength)
- estimateBranchThunks(Call);
- else
- estimateLongThunk(Call);
+ estimateBranchThunks(Call);
+ estimateLongThunk(Call);
}
}
diff --git a/bolt/test/AArch64/relax-call-short-thunk-reuse.s b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
index 54a5b0bf4da6c..031c135fc82b4 100644
--- a/bolt/test/AArch64/relax-call-short-thunk-reuse.s
+++ b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
@@ -84,7 +84,7 @@ E:
.E_entry:
# FDATA: 1 E #.E_entry# 100
ret
- .space 0x80
+ .space 0x78
.size E, .-E
.globl F
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index dfc4a24df3727..0962a594f834a 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -55,22 +55,22 @@
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080456 estimated bytes without thunks
-# CHECK-BOLT-NEXT: BOLT-INFO: 16 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 40 estimated thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: 12 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
-# CHECK-BOLT-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 40 estimated thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: 24 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080440 estimated bytes without thunks
-# CHECK-BOLT-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 56 estimated thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: 16 actual thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
-# CHECK-BOLT-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-NEXT: BOLT-INFO: 40 estimated thunk bytes
# CHECK-BOLT-NEXT: BOLT-INFO: 28 actual thunk bytes
# CHECK-BOLT: BOLT-INFO: relaxed 2 calls with short thunks
# CHECK-BOLT: BOLT-INFO: relaxed 4 calls with long thunks
@@ -84,22 +84,22 @@
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 0
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080440 estimated bytes without thunks
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 40 estimated thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 24 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 1
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 28 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 56 estimated thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 20 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 2
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080456 estimated bytes without thunks
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 32 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 56 estimated thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 20 actual thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: cluster: 3
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 2 fragment(s)
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 88080448 estimated bytes without thunks
-# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 12 estimated thunk bytes
+# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 24 estimated thunk bytes
# CHECK-BOLT-HFE-NEXT: BOLT-INFO: 8 actual thunk bytes
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 3 calls with short thunks
# CHECK-BOLT-HFE: BOLT-INFO: relaxed 2 calls with long thunks
>From bdad9d94d3c46f343d41789e5474edd0fcf5b0f0 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 22 Sep 2026 13:55:24 +0000
Subject: [PATCH 19/22] Preserve tail-call annotations on local trampolines
---
bolt/lib/Passes/LongJmp.cpp | 2 +
.../conditional-tail-call-skipped-target.s | 48 +++++++++++++++++++
2 files changed, 50 insertions(+)
create mode 100644 bolt/test/AArch64/conditional-tail-call-skipped-target.s
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 679e7a8c01a87..83e7d83abf95c 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -991,6 +991,8 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
TrampolineBB = addTrampolineAfter(/*BB=*/nullptr, TargetSymbol,
/*TargetBB=*/nullptr,
/*Count=*/0);
+ if (MIB->isTailCall(Inst))
+ MIB->convertJmpToTailCall(*TrampolineBB->getLastNonPseudoInstr());
SymbolTrampolines[TargetSymbol] = TrampolineBB;
auto L = BC.scopeLock();
MIB->replaceBranchTarget(Inst, TrampolineBB->getLabel(),
diff --git a/bolt/test/AArch64/conditional-tail-call-skipped-target.s b/bolt/test/AArch64/conditional-tail-call-skipped-target.s
new file mode 100644
index 0000000000000..ce67d4ce7b5c6
--- /dev/null
+++ b/bolt/test/AArch64/conditional-tail-call-skipped-target.s
@@ -0,0 +1,48 @@
+## Preserve tail-call annotations on local trampolines, including reused ones.
+## The indirect branch makes _start non-simple. Its conditional tail calls
+## are redirected through a shared trampoline whose outgoing B must also be
+## annotated as a tail call, even when the destination is excluded from layout.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,_start %s -o %t -nostdlib
+# RUN: llvm-bolt %t -o %t.bolt --relocs --lite=0 --relax-exp \
+# RUN: --skip-funcs='^skipped$' | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
+
+# CHECK-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
+# CHECK-BOLT: BOLT-INFO: 1 long thunks created
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ cmp x0, #0
+ b.eq skipped
+ cmp x1, #0
+ b.eq skipped
+ ldr x16, [sp]
+ br x16
+ .size _start, .-_start
+
+ .globl skipped
+ .type skipped, %function
+skipped:
+ ret
+ .size skipped, .-skipped
+
+ .reloc 0, R_AARCH64_NONE
+
+# CHECK-OUTPUT: Disassembly of section .text:
+# CHECK-OUTPUT: <_start>:
+# CHECK-OUTPUT-NEXT: {{.*}} cmp x0, #0x0
+# CHECK-OUTPUT-NEXT: {{.*}} b.eq 0x[[STUB:[0-9a-f]+]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: {{.*}} cmp x1, #0x0
+# CHECK-OUTPUT-NEXT: {{.*}} b.eq 0x[[STUB]] <{{.*}}>
+# CHECK-OUTPUT-NEXT: {{.*}} ldr x16, [sp]
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
+# CHECK-OUTPUT-NEXT: [[STUB]]: {{.*}} b 0x[[THUNK:[0-9a-f]+]] <__AArch64_forward_ADRPThunk_skipped_0>
+# CHECK-OUTPUT: <__AArch64_forward_ADRPThunk_skipped_0>:
+# CHECK-OUTPUT-NEXT: [[THUNK]]: {{.*}} adrp x16,
+# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16,
+# CHECK-OUTPUT-NEXT: {{.*}} br x16
>From d6bd905b7441bf809a3a7a84db47b88cf027cc45 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 22 Sep 2026 14:35:13 +0000
Subject: [PATCH 20/22] clean up run lines
---
.../AArch64/conditional-tail-call-skipped-target.s | 4 ++--
bolt/test/AArch64/relax-branches-with-thunk-chain.s | 5 ++---
bolt/test/AArch64/relax-call-long-thunk-reuse.s | 5 ++---
bolt/test/AArch64/relax-call-short-thunk-reuse.s | 5 ++---
bolt/test/AArch64/relax-calls.s | 11 ++++-------
bolt/test/AArch64/relax-cross-fragment-calls.s | 6 ++----
.../AArch64/tail-classified-body-branch-live-x16.S | 12 ++++--------
7 files changed, 18 insertions(+), 30 deletions(-)
diff --git a/bolt/test/AArch64/conditional-tail-call-skipped-target.s b/bolt/test/AArch64/conditional-tail-call-skipped-target.s
index ce67d4ce7b5c6..71bda8d05813a 100644
--- a/bolt/test/AArch64/conditional-tail-call-skipped-target.s
+++ b/bolt/test/AArch64/conditional-tail-call-skipped-target.s
@@ -6,8 +6,8 @@
# REQUIRES: system-linux
# RUN: %clang %cflags -Wl,-q -Wl,-e,_start %s -o %t -nostdlib
-# RUN: llvm-bolt %t -o %t.bolt --relocs --lite=0 --relax-exp \
-# RUN: --skip-funcs='^skipped$' | FileCheck %s --check-prefix=CHECK-BOLT
+# RUN: llvm-bolt %t -o %t.bolt --relax-exp --skip-funcs='^skipped$' \
+# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
# CHECK-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
diff --git a/bolt/test/AArch64/relax-branches-with-thunk-chain.s b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
index d4c0d5c696630..26527f488ebc6 100644
--- a/bolt/test/AArch64/relax-branches-with-thunk-chain.s
+++ b/bolt/test/AArch64/relax-branches-with-thunk-chain.s
@@ -22,11 +22,10 @@
# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
# RUN: link_fdata --no-lbr %s %t %t.fdata
# RUN: llvm-strip --strip-unneeded %t
-# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions \
-# RUN: --compact-code-model --relax-exp \
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions --relax-exp \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-bolt %t -o %t.hfe.bolt --data %t.fdata --split-functions \
-# RUN: --compact-code-model --relax-exp --hot-functions-at-end \
+# RUN: --relax-exp --hot-functions-at-end \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
diff --git a/bolt/test/AArch64/relax-call-long-thunk-reuse.s b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
index d7436daff83ec..c4f96213c4639 100644
--- a/bolt/test/AArch64/relax-call-long-thunk-reuse.s
+++ b/bolt/test/AArch64/relax-call-long-thunk-reuse.s
@@ -19,9 +19,8 @@
# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
# RUN: link_fdata --no-lbr %s %t %t.fdata
# RUN: llvm-strip --strip-unneeded %t
-# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata \
-# RUN: --compact-code-model --relax-exp --max-cluster-size=64 \
-# RUN: --max-thunk-chain-length=0 \
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --relax-exp \
+# RUN: --max-cluster-size=64 --max-thunk-chain-length=0 \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-objdump -d \
# RUN: --disassemble-symbols=A,B,C,D,E,__AArch64_backward_ADRPThunk_A_0,__AArch64_backward_ADRPThunk_A_1 \
diff --git a/bolt/test/AArch64/relax-call-short-thunk-reuse.s b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
index 031c135fc82b4..3034cf59b50f2 100644
--- a/bolt/test/AArch64/relax-call-short-thunk-reuse.s
+++ b/bolt/test/AArch64/relax-call-short-thunk-reuse.s
@@ -14,9 +14,8 @@
# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
# RUN: link_fdata --no-lbr %s %t %t.fdata
# RUN: llvm-strip --strip-unneeded %t
-# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata \
-# RUN: --compact-code-model --relax-exp --max-cluster-size=160 \
-# RUN: --max-thunk-chain-length=3 \
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --relax-exp \
+# RUN: --max-cluster-size=160 --max-thunk-chain-length=3 \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-objdump -d \
# RUN: --disassemble-symbols=A,B,C,D,E,F,__AArch64_forward_Thunk_F_0,__AArch64_forward_Thunk_F_1,__AArch64_forward_Thunk_F_2 \
diff --git a/bolt/test/AArch64/relax-calls.s b/bolt/test/AArch64/relax-calls.s
index ec89f3ce69f85..a5bebbdef5464 100644
--- a/bolt/test/AArch64/relax-calls.s
+++ b/bolt/test/AArch64/relax-calls.s
@@ -6,15 +6,12 @@
# RUN: link_fdata %s %t.o %t.fdata
# RUN: llvm-strip --strip-unneeded %t.o
# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
-# RUN: --lite=1 --data %t.fdata \
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=1 --data %t.fdata \
# RUN: --print-normalized 2>&1 | FileCheck %s --check-prefix=CHECK-BOLT-LITE
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
-# RUN: --lite=0 --data %t.fdata \
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --lite=0 --data %t.fdata \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
-# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp \
-# RUN: --hot-functions-at-end --lite=0 --data %t.fdata \
-# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HOT-END
+# RUN: llvm-bolt %t.exe -o %t.bolt --relax-exp --hot-functions-at-end --lite=0 \
+# RUN: --data %t.fdata | FileCheck %s --check-prefix=CHECK-BOLT-HOT-END
# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
## Constant islands at the end of functions foo(), bar(), and _start() make each
diff --git a/bolt/test/AArch64/relax-cross-fragment-calls.s b/bolt/test/AArch64/relax-cross-fragment-calls.s
index 0962a594f834a..843b2241f3925 100644
--- a/bolt/test/AArch64/relax-cross-fragment-calls.s
+++ b/bolt/test/AArch64/relax-cross-fragment-calls.s
@@ -36,12 +36,10 @@
# RUN: %clang %cflags -Wl,-q -Wl,-e,A %s -o %t -nostdlib
# RUN: link_fdata --no-lbr %s %t %t.fdata
# RUN: llvm-strip --strip-unneeded %t
-# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions \
-# RUN: --compact-code-model --relax-exp \
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --split-functions --relax-exp \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
# RUN: llvm-bolt %t -o %t.hfe.bolt --data %t.fdata --split-functions \
-# RUN: --compact-code-model --relax-exp \
-# RUN: --hot-functions-at-end \
+# RUN: --relax-exp --hot-functions-at-end \
# RUN: | FileCheck %s --check-prefix=CHECK-BOLT-HFE
# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=CHECK-SECTIONS
# RUN: llvm-objdump -d \
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
index eaae30479decc..f99d503af29bb 100644
--- a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
+++ b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
@@ -10,20 +10,16 @@
# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t.tailcall -nostdlib
# RUN: link_fdata --no-lbr %s %t.tailcall %t.tailcall.fdata
-# RUN: llvm-bolt %t.tailcall -o %t.tailcall.bolt \
-# RUN: --data %t.tailcall.fdata --relocs \
-# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
-# RUN: --skip-funcs='^bar$' \
+# RUN: llvm-bolt %t.tailcall -o %t.tailcall.bolt --data %t.tailcall.fdata \
+# RUN: --relax-exp --reorder-functions=exec-count --skip-funcs='^bar$' \
# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
# RUN: llvm-objdump -d \
# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_ADRPThunk_bar_0 \
# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
-# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch \
-# RUN: -nostdlib
+# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch -nostdlib
# RUN: link_fdata --no-lbr %s %t.branch %t.branch.fdata
-# RUN: llvm-bolt %t.branch -o %t.branch.bolt \
-# RUN: --data %t.branch.fdata --relocs \
+# RUN: llvm-bolt %t.branch -o %t.branch.bolt --data %t.branch.fdata \
# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
# RUN: | FileCheck %s --check-prefix=BRANCH-BOLT
# RUN: llvm-objdump -d \
>From cdee49af0a57468666b652a8fb9419bd28011eb5 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 22 Sep 2026 16:32:05 +0000
Subject: [PATCH 21/22] Revert tail-call annotations on local trampolines.
Remove secondary-entry heuristic from tail-call relaxation This patch is
already doing too much work; we should look at these issues later.
---
bolt/lib/Passes/LongJmp.cpp | 25 ++--
.../conditional-tail-call-skipped-target.s | 48 -------
.../tail-classified-body-branch-live-x16.S | 132 ------------------
3 files changed, 8 insertions(+), 197 deletions(-)
delete mode 100644 bolt/test/AArch64/conditional-tail-call-skipped-target.s
delete mode 100644 bolt/test/AArch64/tail-classified-body-branch-live-x16.S
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 83e7d83abf95c..43b692ff50fb1 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -991,8 +991,6 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
TrampolineBB = addTrampolineAfter(/*BB=*/nullptr, TargetSymbol,
/*TargetBB=*/nullptr,
/*Count=*/0);
- if (MIB->isTailCall(Inst))
- MIB->convertJmpToTailCall(*TrampolineBB->getLastNonPseudoInstr());
SymbolTrampolines[TargetSymbol] = TrampolineBB;
auto L = BC.scopeLock();
MIB->replaceBranchTarget(Inst, TrampolineBB->getLabel(),
@@ -1380,11 +1378,6 @@ void ClusteredRelaxation::printStats() const {
void ClusteredRelaxation::collectOutOfRangeReferences() {
CallsByDistance.resize(Clusters.size());
- auto isPrimaryEntryTarget = [&](const MCSymbol *TargetSymbol) {
- uint64_t EntryID = 0;
- const BinaryFunction *BF = BC.getFunctionForSymbol(TargetSymbol, &EntryID);
- return BF && EntryID == 0;
- };
// Walk all instructions once and collect both branches and calls.
for (BinaryFunction *BF : OutputFunctions) {
@@ -1404,7 +1397,6 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
InstOffset += 4;
const bool IsCall = BC.MIB->isCall(Inst);
- const bool IsTailCall = BC.MIB->isTailCall(Inst);
const bool IsUncondBranch = BC.MIB->isUnconditionalBranch(Inst);
if (!IsCall && !IsUncondBranch)
continue;
@@ -1425,15 +1417,14 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
const OutOfRangeRef Reference{&Inst, TargetSymbol,
SourceOffset, Target.Offset,
Source.Cluster, Target.Cluster};
- const bool UseBranchChain =
- Found && (IsUncondBranch ||
- (IsTailCall && !isPrimaryEntryTarget(TargetSymbol)));
- if (UseBranchChain) {
- // A direct B to a body entry may be annotated as a tail call. It is
- // not an ABI call boundary, so use branch chains instead of call
- // thunks that may clobber x16/x17.
- if (IsTailCall)
- BC.MIB->convertTailCallToJmp(Inst);
+ if (IsUncondBranch) {
+ if (!Found) {
+ BC.errs() << "BOLT-ERROR: cannot relax unconditional branch from "
+ << BF->getPrintName() << " to out-of-layout target "
+ << TargetSymbol->getName() << '\n';
+ BC.printInstruction(BC.errs(), Inst);
+ exit(1);
+ }
Branches.push_back(Reference);
continue;
diff --git a/bolt/test/AArch64/conditional-tail-call-skipped-target.s b/bolt/test/AArch64/conditional-tail-call-skipped-target.s
deleted file mode 100644
index 71bda8d05813a..0000000000000
--- a/bolt/test/AArch64/conditional-tail-call-skipped-target.s
+++ /dev/null
@@ -1,48 +0,0 @@
-## Preserve tail-call annotations on local trampolines, including reused ones.
-## The indirect branch makes _start non-simple. Its conditional tail calls
-## are redirected through a shared trampoline whose outgoing B must also be
-## annotated as a tail call, even when the destination is excluded from layout.
-
-# REQUIRES: system-linux
-
-# RUN: %clang %cflags -Wl,-q -Wl,-e,_start %s -o %t -nostdlib
-# RUN: llvm-bolt %t -o %t.bolt --relax-exp --skip-funcs='^skipped$' \
-# RUN: | FileCheck %s --check-prefix=CHECK-BOLT
-# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-OUTPUT
-
-# CHECK-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
-# CHECK-BOLT: BOLT-INFO: 1 long thunks created
-
- .text
- .globl _start
- .type _start, %function
-_start:
- cmp x0, #0
- b.eq skipped
- cmp x1, #0
- b.eq skipped
- ldr x16, [sp]
- br x16
- .size _start, .-_start
-
- .globl skipped
- .type skipped, %function
-skipped:
- ret
- .size skipped, .-skipped
-
- .reloc 0, R_AARCH64_NONE
-
-# CHECK-OUTPUT: Disassembly of section .text:
-# CHECK-OUTPUT: <_start>:
-# CHECK-OUTPUT-NEXT: {{.*}} cmp x0, #0x0
-# CHECK-OUTPUT-NEXT: {{.*}} b.eq 0x[[STUB:[0-9a-f]+]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: {{.*}} cmp x1, #0x0
-# CHECK-OUTPUT-NEXT: {{.*}} b.eq 0x[[STUB]] <{{.*}}>
-# CHECK-OUTPUT-NEXT: {{.*}} ldr x16, [sp]
-# CHECK-OUTPUT-NEXT: {{.*}} br x16
-# CHECK-OUTPUT-NEXT: [[STUB]]: {{.*}} b 0x[[THUNK:[0-9a-f]+]] <__AArch64_forward_ADRPThunk_skipped_0>
-# CHECK-OUTPUT: <__AArch64_forward_ADRPThunk_skipped_0>:
-# CHECK-OUTPUT-NEXT: [[THUNK]]: {{.*}} adrp x16,
-# CHECK-OUTPUT-NEXT: {{.*}} add x16, x16,
-# CHECK-OUTPUT-NEXT: {{.*}} br x16
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
deleted file mode 100644
index f99d503af29bb..0000000000000
--- a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
+++ /dev/null
@@ -1,132 +0,0 @@
-## Check that a direct B to a function-body symbol is not relaxed as an
-## ABI tail call. Such a branch can carry live caller-saved registers, so
-## relaxation must use B-only branch chains instead of ADRP/BR call thunks.
-##
-## In the TAILCALL run, bar_body is at the function entry and the branch is an
-## ABI tail call. In the BRANCH run, bar_body is inside the function body and
-## must be relaxed as a branch.
-
-# REQUIRES: system-linux
-
-# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t.tailcall -nostdlib
-# RUN: link_fdata --no-lbr %s %t.tailcall %t.tailcall.fdata
-# RUN: llvm-bolt %t.tailcall -o %t.tailcall.bolt --data %t.tailcall.fdata \
-# RUN: --relax-exp --reorder-functions=exec-count --skip-funcs='^bar$' \
-# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
-# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_ADRPThunk_bar_0 \
-# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
-
-# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch -nostdlib
-# RUN: link_fdata --no-lbr %s %t.branch %t.branch.fdata
-# RUN: llvm-bolt %t.branch -o %t.branch.bolt --data %t.branch.fdata \
-# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
-# RUN: | FileCheck %s --check-prefix=BRANCH-BOLT
-# RUN: llvm-objdump -d \
-# RUN: --disassemble-symbols=foo,bar,bar_body,.bar_ret,padding,__AArch64_forward_Thunk_0 \
-# RUN: %t.branch.bolt | FileCheck %s --check-prefix=BRANCH
-
-# TAILCALL-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
-# TAILCALL-BOLT: BOLT-INFO: starting experimental relaxation pass
-# TAILCALL-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
-# TAILCALL-BOLT: BOLT-INFO: 1 long thunks created
-
-# BRANCH-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
-# BRANCH-BOLT: BOLT-INFO: starting experimental relaxation pass
-# BRANCH-BOLT: BOLT-INFO: relaxed 1 unconditional branches
-# BRANCH-BOLT: BOLT-INFO: 1 branch thunks created
-
- .text
- .globl bar
- .type bar, %function
-bar:
-#ifdef BRANCH
-.bar_entry:
- b .bar_ret
-#endif
- .globl bar_body
- .type bar_body, %notype
-bar_body:
- cmp x16, #42
- cset w0, ne
- ret
-#ifdef BRANCH
-.bar_ret:
- ret
-#endif
- .size bar, .-bar
-
- .globl foo
- .type foo, %function
-foo:
-.foo_entry:
-# FDATA: 1 foo #.foo_entry# 100
- mov x16, #42
- b bar_body
- .space 0x4400000
- .size foo, .-foo
-
- .globl padding
- .type padding, %function
-padding:
-.padding_entry:
-# FDATA: 1 padding #.padding_entry# 100
- ret
- .space 0x4400000
- .size padding, .-padding
-
-## Force relocation mode.
- .reloc 0, R_AARCH64_NONE
-
-# TAILCALL: Disassembly of section .bolt.org.text:
-#
-# TAILCALL: <bar>:
-# TAILCALL-NEXT: <bar_body>:
-# TAILCALL-NEXT: {{.*}} cmp x16, #0x2a
-# TAILCALL-NEXT: {{.*}} cset w0, ne
-# TAILCALL-NEXT: {{.*}} ret
-#
-# TAILCALL: Disassembly of section .text:
-#
-# TAILCALL: <foo>:
-# TAILCALL-NEXT: {{.*}} mov x16, #0x2a
-# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_ADRPThunk_bar_0>
-#
-# TAILCALL: <__AArch64_forward_ADRPThunk_bar_0>:
-# TAILCALL-NEXT: {{.*}} adrp x16,
-# TAILCALL-NEXT: {{.*}} add x16, x16,
-# TAILCALL-NEXT: {{.*}} br x16
-#
-# TAILCALL: <padding>:
-# TAILCALL-NEXT: {{.*}} ret
-
-
-# BRANCH: Disassembly of section .bolt.org.text:
-#
-# BRANCH: <bar_body>:
-# BRANCH-NEXT: {{.*}} cmp x16, #0x2a
-# BRANCH-NEXT: {{.*}} cset w0, ne
-# BRANCH-NEXT: {{.*}} ret
-#
-# BRANCH: Disassembly of section .text:
-#
-# BRANCH: <foo>:
-# BRANCH-NEXT: {{.*}} mov x16, #0x2a
-# BRANCH-NEXT: {{.*}} b 0x[[CHAIN:[0-9a-f]+]] <__AArch64_forward_Thunk_0>
-#
-# BRANCH: <__AArch64_forward_Thunk_0>:
-# BRANCH-NEXT: [[CHAIN]]: {{.*}} b 0x[[BODY:[0-9a-f]+]] <bar+0x4>
-#
-# BRANCH: <padding>:
-# BRANCH-NEXT: {{.*}} ret
-#
-# BRANCH: Disassembly of section .text.cold:
-#
-# BRANCH: <bar>:
-# BRANCH-NEXT: {{.*}} b 0x[[RET:[0-9a-f]+]] <.bar_ret>
-# BRANCH-NEXT: [[BODY]]: {{.*}} cmp x16, #0x2a
-# BRANCH-NEXT: {{.*}} cset w0, ne
-# BRANCH-NEXT: {{.*}} ret
-#
-# BRANCH: <.bar_ret>:
-# BRANCH-NEXT: [[RET]]: {{.*}} ret
>From c6a879a3ccb8fc50ea851cba7725bfcfc02dd44b Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Thu, 24 Sep 2026 08:14:20 +0000
Subject: [PATCH 22/22] warn for unconditional branches out of layout instead
of error
---
bolt/lib/Passes/LongJmp.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 43b692ff50fb1..3dab645cfd61a 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -1419,11 +1419,11 @@ void ClusteredRelaxation::collectOutOfRangeReferences() {
Source.Cluster, Target.Cluster};
if (IsUncondBranch) {
if (!Found) {
- BC.errs() << "BOLT-ERROR: cannot relax unconditional branch from "
+ BC.errs() << "BOLT-WARNING: cannot relax unconditional branch from "
<< BF->getPrintName() << " to out-of-layout target "
<< TargetSymbol->getName() << '\n';
BC.printInstruction(BC.errs(), Inst);
- exit(1);
+ continue;
}
Branches.push_back(Reference);
More information about the llvm-commits
mailing list