[llvm] [BOLT] Model emitted code sections in LongJmp layout (PR #218232)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 04:49:50 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-bolt
Author: Adam Bzowski (adam-bzowski-arm)
<details>
<summary>Changes</summary>
Replace LongJmp’s coarse hot/cold size calculation with a section-aware layout model that closely follows BinaryEmitter and RewriteInstance. This accounts for section ordering, alignment, function fragments, padding, constant islands, injected functions, and special placement options, preventing missed Branch26 relaxations near the 128 MiB boundary.
---
Patch is 92.20 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218232.diff
24 Files Affected:
- (modified) bolt/include/bolt/Core/BinaryContext.h (+4)
- (modified) bolt/include/bolt/Core/BinaryEmitter.h (+24)
- (modified) bolt/include/bolt/Core/BinaryFunction.h (+3-1)
- (modified) bolt/include/bolt/Passes/LongJmp.h (+106-26)
- (modified) bolt/lib/Core/BinaryContext.cpp (+42)
- (modified) bolt/lib/Core/BinaryEmitter.cpp (+33-19)
- (modified) bolt/lib/Passes/LongJmp.cpp (+482-152)
- (modified) bolt/lib/Passes/ReorderFunctions.cpp (+1-3)
- (modified) bolt/lib/Rewrite/RewriteInstance.cpp (+60-55)
- (added) bolt/test/AArch64/LongJmp-Layout/basic.s (+106)
- (added) bolt/test/AArch64/LongJmp-Layout/constant-island.s (+61)
- (added) bolt/test/AArch64/LongJmp-Layout/empty-function.s (+99)
- (added) bolt/test/AArch64/LongJmp-Layout/fixed-injected.s (+31)
- (added) bolt/test/AArch64/LongJmp-Layout/function-padding.s (+55)
- (added) bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s (+105)
- (added) bolt/test/AArch64/LongJmp-Layout/hugify.s (+48)
- (added) bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s (+39)
- (added) bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s (+33)
- (added) bolt/test/AArch64/LongJmp-Layout/sections.s (+91)
- (added) bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s (+84)
- (added) bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s (+44)
- (added) bolt/test/AArch64/LongJmp-Layout/skip-function.s (+61)
- (modified) bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s (+1-1)
- (modified) bolt/test/AArch64/split-funcs-lite.s (+2-2)
``````````diff
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 310235504663b..add655a912917 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -1169,6 +1169,10 @@ class BinaryContext {
return ".text.injected.cold";
}
+ /// Return true if code section \p A precedes code section \p B in the
+ /// output.
+ bool isCodeSectionBefore(StringRef A, StringRef B) const;
+
ErrorOr<BinarySection &> getGdbIndexSection() const {
return getUniqueSectionByName(".gdb_index");
}
diff --git a/bolt/include/bolt/Core/BinaryEmitter.h b/bolt/include/bolt/Core/BinaryEmitter.h
index 8c3e4b8b64cee..969f492cd1f08 100644
--- a/bolt/include/bolt/Core/BinaryEmitter.h
+++ b/bolt/include/bolt/Core/BinaryEmitter.h
@@ -15,6 +15,7 @@
#define BOLT_CORE_BINARY_EMITTER_H
#include "llvm/ADT/StringRef.h"
+#include <cstddef>
namespace llvm {
class MCStreamer;
@@ -24,6 +25,11 @@ class BinaryContext;
class BinaryFunction;
class FunctionFragment;
+/// Return whether BinaryEmitter::emitFunction() proceeds to select an output
+/// section for \p Function.
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function);
+
/// Emit all code and data from the BinaryContext \p BC into the \p Streamer.
///
/// \p OrgSecPrefix is used to modify name of emitted original sections
@@ -40,4 +46,22 @@ void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
} // namespace bolt
} // namespace llvm
+namespace opts {
+
+/// Return the number of bytes emitted before each fragment of \p Function by
+/// --break-funcs.
+std::size_t breakFunctionSize(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the bytes emitted after each fragment by --mark-funcs.
+llvm::StringRef
+markFunctionBytes(const llvm::bolt::BinaryContext &BinaryContext);
+
+/// Return the padding requested before each emitted fragment of \p Function.
+std::size_t padFunctionBefore(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the padding requested after each emitted fragment of \p Function.
+std::size_t padFunctionAfter(const llvm::bolt::BinaryFunction &Function);
+
+} // namespace opts
+
#endif
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 14d7f9b5b5359..6e24dd73ac37b 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -2314,7 +2314,9 @@ class BinaryFunction {
if (!OnBehalfOf) {
for (BinaryFunction *ExternalFunc : Islands->Dependency) {
- Size = alignTo(Size, ExternalFunc->getConstantIslandAlignment());
+ // BinaryEmitter aligns dependency islands using the constant-island
+ // alignment of the host function.
+ Size = alignTo(Size, getConstantIslandAlignment());
Size += ExternalFunc->estimateConstantIslandSize(this);
}
}
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index fa8cf67a95d73..d0eaace481695 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -21,15 +21,15 @@ class BranchLivenessInfo;
/// pull this pass inside BOLT because here we can do a better job at stub
/// inserting by manipulating the CFG, something linkers can't do.
///
-/// We iteratively repeat the following until no modification is done: we do a
-/// tentative layout with the current function sizes; then we add stubs for
+/// We iteratively repeat the following until no modification is done: we
+/// compute the layout with the current function sizes; then we add stubs for
/// branches that we know are out of range or we expand smaller stubs (28-bit)
/// to a large one if necessary (32 or 64).
///
-/// This expansion inserts the equivalent of "linker stubs", small
-/// blocks of code that load a 64-bit address into a pre-allocated register and
-// then executes an unconditional indirect branch on this register. By using a
-/// 64-bit range, we guarantee it can reach any code location.
+/// This expansion inserts the equivalent of "linker stubs": small blocks of
+/// code that load a 64-bit address into a pre-allocated register and then
+/// execute an unconditional indirect branch through that register. By using a
+/// 64-bit range, we guarantee that they can reach any code location.
///
class LongJmpPass : public BinaryFunctionPass {
/// Used to implement stub grouping (reusing a stub from one function into
@@ -52,9 +52,10 @@ class LongJmpPass : public BinaryFunctionPass {
DenseMap<const BinaryFunction *, std::set<const BinaryBasicBlock *>> Stubs;
using FuncAddressesMapTy = DenseMap<const BinaryFunction *, uint64_t>;
- /// Hold tentative addresses
+ /// Main-fragment start addresses for the current layout iteration.
FuncAddressesMapTy HotAddresses;
- FuncAddressesMapTy ColdAddresses;
+
+ /// Basic-block start addresses for the current layout iteration.
DenseMap<const BinaryBasicBlock *, uint64_t> BBAddresses;
/// Used to identify the stub size
@@ -121,23 +122,102 @@ class LongJmpPass : public BinaryFunctionPass {
/// Relax calls using function cluster approach.
void relaxCalls(BinaryContext &BC);
- /// -- Layout estimation methods --
- /// Try to do layout before running the emitter, by looking at BinaryFunctions
- /// and MCInsts -- this is an estimation. To be correct for longjmp inserter
- /// purposes, we need to do a size worst-case estimation. Real layout is done
- /// by RewriteInstance::mapFileSections()
- void tentativeLayout(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions);
- uint64_t tentativeLayoutRelocMode(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- uint64_t tentativeLayoutRelocColdPart(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- void tentativeBBLayout(const BinaryFunction &Func);
-
- /// Update stubs addresses with their exact address after a round of stub
- /// insertion and layout estimation is done.
+ /// Identifies a function fragment and its owning function.
+ struct FunctionFragmentPlacement {
+ /// Function that owns the fragment.
+ const BinaryFunction *Func;
+
+ /// Number identifying the fragment within the function layout.
+ FragmentNum Fragment;
+ };
+
+ /// Describes the placement of one emitted code section.
+ struct SectionPlacement {
+ /// Output section name returned by BinaryFunction::getCodeSectionName().
+ SmallString<32> Name;
+
+ /// Fragments in BinaryEmitter::emitFunctions() emission order.
+ SmallVector<FunctionFragmentPlacement, 0> Fragments;
+
+ /// Maximum alignment required by the section and its contents.
+ uint64_t Alignment;
+ };
+
+ /// Code sections whose addresses are determined during final mapping, and
+ /// their fragments. In relocation mode, sections are sorted like
+ /// RewriteInstance::getCodeSections() before allocation. In non-relocation
+ /// mode, these are the non-fixed injected sections in emission order.
+ SmallVector<SectionPlacement, 4> Sections;
+
+ /// Update \p CurrentAlignment with the requirements added while
+ /// BinaryEmitter emits \p FF. The non-relocation path is used only for
+ /// non-fixed injected sections.
+ uint64_t updateSectionAlignment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t CurrentAlignment) const;
+
+ /// Assign \p FF to its output section, creating its placement if needed.
+ void assignFunctionFragmentToSection(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF);
+
+ /// Assign fragments to output sections in BinaryEmitter::emitFunctions()
+ /// order. In relocation mode, collect all emitted non-fixed fragments. In
+ /// non-relocation mode, collect only non-fixed injected functions. Functions
+ /// mapped independently by RewriteInstance are excluded.
+ void assignFunctionsToSections(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Mirror the code-section contents emitted by
+ /// BinaryEmitter::emitFunctionBody(). Labels, CFI and debug directives do not
+ /// advance the address and are omitted. If \p RecordAddresses is false, only
+ /// calculate the ending address.
+ uint64_t layoutFunctionBody(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Mirror the code-section contents emitted by BinaryEmitter::emitFunction().
+ /// \p Func must pass shouldEmitFunctionFragment(). If \p RecordAddresses is
+ /// false, only calculate the ending address.
+ uint64_t layoutFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out the fragments in \p Section in emission order and return the first
+ /// address after the section. If \p RecordAddresses is false, do not update
+ /// the function or basic-block address maps.
+ uint64_t layoutSection(const BinaryContext &BC,
+ const SectionPlacement &Section, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out Sections toward increasing addresses and return the first address
+ /// after them. This mirrors allocateAt() in relocation mode and the injected
+ /// section allocation in mapCodeSectionsInPlace() otherwise.
+ uint64_t layoutSectionsForward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Mirror the relocation-mode allocateBefore() helper in
+ /// RewriteInstance::mapCodeSections(). Calculate section sizes while
+ /// allocating before \p DotAddress. Return false if subtraction would
+ /// underflow or alignment would place a section before the start of old
+ /// .text (BC.OldTextSectionAddress); otherwise record the layout and return
+ /// true.
+ bool layoutSectionsBackward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Lay out section-mapped and independently placed functions according to
+ /// the mapping rules for the current relocation mode.
+ void layoutFunctions(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Compute the code layout for the current LongJmp iteration by mirroring
+ /// BinaryEmitter emission and RewriteInstance section mapping.
+ void layout(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Update stub addresses after computing the current layout.
void updateStubGroups();
/// -- Relaxation/stub insertion methods --
@@ -189,7 +269,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Expand the range of the stub in StubBB if necessary
Error relaxStub(BinaryBasicBlock &StubBB, bool &Modified);
- /// Helper to resolve a symbol address according to our tentative layout
+ /// Helper to resolve a symbol address according to our computed layout.
uint64_t getSymbolAddress(const BinaryContext &BC, const MCSymbol *Target,
const BinaryBasicBlock *TgtBB) const;
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 47d463d3f7f6f..19373c5729e45 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -342,6 +342,48 @@ bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {
return false;
}
+bool BinaryContext::isCodeSectionBefore(StringRef A, StringRef B) const {
+ if (A == B)
+ return false;
+
+ // If both A and B have names starting with ".text.cold", then
+ // - if opts::HotFunctionsAtEnd is true, we want order
+ // ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
+ // - if opts::HotFunctionsAtEnd is false, we want order
+ // ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
+ if (A.starts_with(getColdCodeSectionName()) &&
+ B.starts_with(getColdCodeSectionName())) {
+ if (A.size() != B.size())
+ return opts::HotFunctionsAtEnd ? A.size() > B.size()
+ : A.size() < B.size();
+ return opts::HotFunctionsAtEnd ? A > B : A < B;
+ }
+
+ // Place hot text movers before anything else.
+ if (opts::HotText) {
+ if (A == getHotTextMoverSectionName())
+ return true;
+ if (B == getHotTextMoverSectionName())
+ return false;
+ }
+
+ // Depending on opts::HotFunctionsAtEnd, place main and warm sections in
+ // order.
+ if (opts::HotFunctionsAtEnd) {
+ if (B == getMainCodeSectionName())
+ return true;
+ if (A == getMainCodeSectionName())
+ return false;
+ return B == getWarmCodeSectionName();
+ }
+
+ if (A == getMainCodeSectionName())
+ return true;
+ if (B == getMainCodeSectionName())
+ return false;
+ return A == getWarmCodeSectionName();
+}
+
std::unique_ptr<MCObjectWriter>
BinaryContext::createObjectWriter(raw_pwrite_stream &OS) {
return MAB->createObjectWriter(OS);
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 9243b6701b918..36add8cfda079 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -88,10 +88,22 @@ size_t padFunction(std::map<std::string, size_t> &FunctionPadding,
return 0;
}
+size_t breakFunctionSize(const BinaryFunction &Function) {
+ for (const std::string &Name : BreakFunctionNames)
+ if (Function.hasNameRegex(Name))
+ return 2;
+ return 0;
+}
+
+StringRef markFunctionBytes(const BinaryContext &BC) {
+ return MarkFuncs ? BC.MIB->getTrapFillValue() : StringRef();
+}
+
size_t padFunctionBefore(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadBeforeSpec, Function);
}
+
size_t padFunctionAfter(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadSpec, Function);
@@ -286,15 +298,7 @@ void BinaryEmitter::emitFunctions() {
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
FunctionFragment &FF) {
- if (Function.size() == 0 && !Function.hasIslandsInfo())
- return false;
-
- if (Function.getState() == BinaryFunction::State::Empty)
- return false;
-
- // Avoid emitting function without instructions when overwriting the original
- // function in-place. Otherwise, emit the empty function to define the symbol.
- if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ if (!shouldEmitFunctionFragment(BC, Function))
return false;
MCSection *Section =
@@ -387,14 +391,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
"first basic block should never be cold");
// Emit UD2 at the beginning if requested by user.
- if (!opts::BreakFunctionNames.empty()) {
- for (std::string &Name : opts::BreakFunctionNames) {
- if (Function.hasNameRegex(Name)) {
- Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
- break;
- }
- }
- }
+ if (const size_t BreakSize = opts::breakFunctionSize(Function))
+ Streamer.emitIntValue(0x0B0F, BreakSize); // UD2: 0F 0B
// Emit code.
emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
@@ -406,8 +404,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
Streamer.emitFill(Padding, MAI.getTextAlignFillValue());
}
- if (opts::MarkFuncs)
- Streamer.emitBytes(BC.MIB->getTrapFillValue());
+ if (StringRef Marker = opts::markFunctionBytes(BC); !Marker.empty())
+ Streamer.emitBytes(Marker);
// Emit CFI end
if (NeedsFDE)
@@ -1226,6 +1224,22 @@ void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
namespace llvm {
namespace bolt {
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function) {
+ if (Function.size() == 0 && !Function.hasIslandsInfo())
+ return false;
+
+ if (Function.getState() == BinaryFunction::State::Empty)
+ return false;
+
+ // Avoid emitting function without instructions when overwriting the original
+ // function in-place. Otherwise, emit the empty function to define the symbol.
+ if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ return false;
+
+ return true;
+}
+
void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
StringRef OrgSecPrefix) {
BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 30ff9dc4ddc71..d528e110984a7 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -11,12 +11,16 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/LongJmp.h"
+#include "bolt/Core/BinaryEmitter.h"
+#include "bolt/Core/FunctionLayout.h"
#include "bolt/Core/ParallelUtilities.h"
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
+#include <cstdint>
+#include <optional>
#define DEBUG_TYPE "longjmp"
@@ -26,7 +30,6 @@ namespace opts {
extern cl::OptionCategory BoltCategory;
extern cl::OptionCategory BoltOptCategory;
extern cl::opt<bool> UseOldText;
-extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool> GroupStubs("group-stubs",
cl::desc("share stubs across functions"),
@@ -40,12 +43,12 @@ static cl::opt<bool>
static cl::opt<bool> RelaxPLT("relax-plt",
cl::desc("indicate PLT proximity to hot text"),
cl::init(true), cl::cat(BoltOptCategory));
-}
+} // namespace opts
namespace llvm {
namespace bolt {
-constexpr unsigned ColdFragAlign = 16;
+static const Align ColdFragmentAlignment(16);
static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();
@@ -124,6 +127,12 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
Stubs[&Func].insert(StubBB.get());
StubBits[StubBB.get()] = BC.MIB->getUncondBranchEncodingSize();
+ LLVM_DEBUG(
+ dbgs() << "BOLT-DEBUG: LongJmp: creating " << (IsCold ? "cold" : "main")
+ << " stub " << StubSym->getName() << " in " << Func.getPrintName()
+ << " at current layout address 0x" << Twine::utohexstr(AtAddress)
+ << " for " << TgtSym->getName() << '\n');
+
if (IsCold) {
registerInMap(ColdLocalStubs[&Func]);
if (opts::GroupStubs && TgtIsFunc)
@@ -172,10 +181,10 @@ BinaryBasicBlock *LongJmpPass::lookupStubFromGroup(
LLVM_DEBUG({
if (Candidates.size() > 1)
- dbgs() << "Considering stub group with " << Candidates.size()
- << " candidates. DotAddress is " << Twine::utohexstr(DotAddress)
- << ", chosen candidate address is "
- << Twine::utohexstr(Cand->first) << "\n";
+ dbgs() << "BOLT-DEBUG: LongJmp: considering stub group with "
+ << Candidates.size() << " candidates at 0x"
+ << Twine::utohexstr(DotAddress) << "; selected candidate at 0x"
+ << Twine::utohexstr(Cand->first) << '\n';
});
return (PCOffset < MinVal || PCOffset > MaxVal) ? nullptr : Cand->second;
}
@@ -294,7 +303,7 @@ void LongJmpPass::updateStubGroups() {
auto update = [&](StubGroupsTy &StubGroups) {
for (auto &KeyVal : StubGroups) {
for (StubTy &Elem : KeyVal.second)
- Elem.first = BBAddresses[Elem.second];
+ Elem.first = BBAddresses.at(Elem.seco...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218232
More information about the llvm-commits
mailing list