[llvm] [BOLT] Model emitted code sections in LongJmp layout (PR #218232)
Adam Bzowski via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 04:49:12 PDT 2026
https://github.com/adam-bzowski-arm created https://github.com/llvm/llvm-project/pull/218232
Replace LongJmp’s coarse hot/cold size calculation with a section-aware layout model that closely follows BinaryEmitter and RewriteInstance. This accounts for section ordering, alignment, function fragments, padding, constant islands, injected functions, and special placement options, preventing missed Branch26 relaxations near the 128 MiB boundary.
>From fb12eb840bcd27489b088b44927b5c609dba270d Mon Sep 17 00:00:00 2001
From: Adam Bzowski <Adam.Bzowski at arm.com>
Date: Sun, 23 Aug 2026 11:29:52 +0000
Subject: [PATCH] [BOLT] Model emitted code sections in LongJmp layout
Change-Id: I40f0d82d3f24a498e1026ee77bb664885a4e9fdd
---
bolt/include/bolt/Core/BinaryContext.h | 4 +
bolt/include/bolt/Core/BinaryEmitter.h | 24 +
bolt/include/bolt/Core/BinaryFunction.h | 4 +-
bolt/include/bolt/Passes/LongJmp.h | 132 +++-
bolt/lib/Core/BinaryContext.cpp | 42 ++
bolt/lib/Core/BinaryEmitter.cpp | 52 +-
bolt/lib/Passes/LongJmp.cpp | 634 +++++++++++++-----
bolt/lib/Passes/ReorderFunctions.cpp | 4 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 115 ++--
bolt/test/AArch64/LongJmp-Layout/basic.s | 106 +++
.../AArch64/LongJmp-Layout/constant-island.s | 61 ++
.../AArch64/LongJmp-Layout/empty-function.s | 99 +++
.../AArch64/LongJmp-Layout/fixed-injected.s | 31 +
.../AArch64/LongJmp-Layout/function-padding.s | 55 ++
.../LongJmp-Layout/hot-functions-at-end.s | 105 +++
bolt/test/AArch64/LongJmp-Layout/hugify.s | 48 ++
.../AArch64/LongJmp-Layout/injected-nonrel.s | 39 ++
.../LongJmp-Layout/non-simple-nonrel.s | 33 +
bolt/test/AArch64/LongJmp-Layout/sections.s | 91 +++
.../LongJmp-Layout/skip-function-frontier.s | 84 +++
.../LongJmp-Layout/skip-function-padding.s | 44 ++
.../AArch64/LongJmp-Layout/skip-function.s | 61 ++
.../long-jmp-hugify-fixup-out-of-range.s | 2 +-
bolt/test/AArch64/split-funcs-lite.s | 4 +-
24 files changed, 1615 insertions(+), 259 deletions(-)
create mode 100644 bolt/test/AArch64/LongJmp-Layout/basic.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/constant-island.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/empty-function.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/function-padding.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/hugify.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/sections.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function.s
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 310235504663b..add655a912917 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -1169,6 +1169,10 @@ class BinaryContext {
return ".text.injected.cold";
}
+ /// Return true if code section \p A precedes code section \p B in the
+ /// output.
+ bool isCodeSectionBefore(StringRef A, StringRef B) const;
+
ErrorOr<BinarySection &> getGdbIndexSection() const {
return getUniqueSectionByName(".gdb_index");
}
diff --git a/bolt/include/bolt/Core/BinaryEmitter.h b/bolt/include/bolt/Core/BinaryEmitter.h
index 8c3e4b8b64cee..969f492cd1f08 100644
--- a/bolt/include/bolt/Core/BinaryEmitter.h
+++ b/bolt/include/bolt/Core/BinaryEmitter.h
@@ -15,6 +15,7 @@
#define BOLT_CORE_BINARY_EMITTER_H
#include "llvm/ADT/StringRef.h"
+#include <cstddef>
namespace llvm {
class MCStreamer;
@@ -24,6 +25,11 @@ class BinaryContext;
class BinaryFunction;
class FunctionFragment;
+/// Return whether BinaryEmitter::emitFunction() proceeds to select an output
+/// section for \p Function.
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function);
+
/// Emit all code and data from the BinaryContext \p BC into the \p Streamer.
///
/// \p OrgSecPrefix is used to modify name of emitted original sections
@@ -40,4 +46,22 @@ void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
} // namespace bolt
} // namespace llvm
+namespace opts {
+
+/// Return the number of bytes emitted before each fragment of \p Function by
+/// --break-funcs.
+std::size_t breakFunctionSize(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the bytes emitted after each fragment by --mark-funcs.
+llvm::StringRef
+markFunctionBytes(const llvm::bolt::BinaryContext &BinaryContext);
+
+/// Return the padding requested before each emitted fragment of \p Function.
+std::size_t padFunctionBefore(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the padding requested after each emitted fragment of \p Function.
+std::size_t padFunctionAfter(const llvm::bolt::BinaryFunction &Function);
+
+} // namespace opts
+
#endif
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 14d7f9b5b5359..6e24dd73ac37b 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -2314,7 +2314,9 @@ class BinaryFunction {
if (!OnBehalfOf) {
for (BinaryFunction *ExternalFunc : Islands->Dependency) {
- Size = alignTo(Size, ExternalFunc->getConstantIslandAlignment());
+ // BinaryEmitter aligns dependency islands using the constant-island
+ // alignment of the host function.
+ Size = alignTo(Size, getConstantIslandAlignment());
Size += ExternalFunc->estimateConstantIslandSize(this);
}
}
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index fa8cf67a95d73..d0eaace481695 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -21,15 +21,15 @@ class BranchLivenessInfo;
/// pull this pass inside BOLT because here we can do a better job at stub
/// inserting by manipulating the CFG, something linkers can't do.
///
-/// We iteratively repeat the following until no modification is done: we do a
-/// tentative layout with the current function sizes; then we add stubs for
+/// We iteratively repeat the following until no modification is done: we
+/// compute the layout with the current function sizes; then we add stubs for
/// branches that we know are out of range or we expand smaller stubs (28-bit)
/// to a large one if necessary (32 or 64).
///
-/// This expansion inserts the equivalent of "linker stubs", small
-/// blocks of code that load a 64-bit address into a pre-allocated register and
-// then executes an unconditional indirect branch on this register. By using a
-/// 64-bit range, we guarantee it can reach any code location.
+/// This expansion inserts the equivalent of "linker stubs": small blocks of
+/// code that load a 64-bit address into a pre-allocated register and then
+/// execute an unconditional indirect branch through that register. By using a
+/// 64-bit range, we guarantee that they can reach any code location.
///
class LongJmpPass : public BinaryFunctionPass {
/// Used to implement stub grouping (reusing a stub from one function into
@@ -52,9 +52,10 @@ class LongJmpPass : public BinaryFunctionPass {
DenseMap<const BinaryFunction *, std::set<const BinaryBasicBlock *>> Stubs;
using FuncAddressesMapTy = DenseMap<const BinaryFunction *, uint64_t>;
- /// Hold tentative addresses
+ /// Main-fragment start addresses for the current layout iteration.
FuncAddressesMapTy HotAddresses;
- FuncAddressesMapTy ColdAddresses;
+
+ /// Basic-block start addresses for the current layout iteration.
DenseMap<const BinaryBasicBlock *, uint64_t> BBAddresses;
/// Used to identify the stub size
@@ -121,23 +122,102 @@ class LongJmpPass : public BinaryFunctionPass {
/// Relax calls using function cluster approach.
void relaxCalls(BinaryContext &BC);
- /// -- Layout estimation methods --
- /// Try to do layout before running the emitter, by looking at BinaryFunctions
- /// and MCInsts -- this is an estimation. To be correct for longjmp inserter
- /// purposes, we need to do a size worst-case estimation. Real layout is done
- /// by RewriteInstance::mapFileSections()
- void tentativeLayout(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions);
- uint64_t tentativeLayoutRelocMode(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- uint64_t tentativeLayoutRelocColdPart(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- void tentativeBBLayout(const BinaryFunction &Func);
-
- /// Update stubs addresses with their exact address after a round of stub
- /// insertion and layout estimation is done.
+ /// Identifies a function fragment and its owning function.
+ struct FunctionFragmentPlacement {
+ /// Function that owns the fragment.
+ const BinaryFunction *Func;
+
+ /// Number identifying the fragment within the function layout.
+ FragmentNum Fragment;
+ };
+
+ /// Describes the placement of one emitted code section.
+ struct SectionPlacement {
+ /// Output section name returned by BinaryFunction::getCodeSectionName().
+ SmallString<32> Name;
+
+ /// Fragments in BinaryEmitter::emitFunctions() emission order.
+ SmallVector<FunctionFragmentPlacement, 0> Fragments;
+
+ /// Maximum alignment required by the section and its contents.
+ uint64_t Alignment;
+ };
+
+ /// Code sections whose addresses are determined during final mapping, and
+ /// their fragments. In relocation mode, sections are sorted like
+ /// RewriteInstance::getCodeSections() before allocation. In non-relocation
+ /// mode, these are the non-fixed injected sections in emission order.
+ SmallVector<SectionPlacement, 4> Sections;
+
+ /// Update \p CurrentAlignment with the requirements added while
+ /// BinaryEmitter emits \p FF. The non-relocation path is used only for
+ /// non-fixed injected sections.
+ uint64_t updateSectionAlignment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t CurrentAlignment) const;
+
+ /// Assign \p FF to its output section, creating its placement if needed.
+ void assignFunctionFragmentToSection(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF);
+
+ /// Assign fragments to output sections in BinaryEmitter::emitFunctions()
+ /// order. In relocation mode, collect all emitted non-fixed fragments. In
+ /// non-relocation mode, collect only non-fixed injected functions. Functions
+ /// mapped independently by RewriteInstance are excluded.
+ void assignFunctionsToSections(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Mirror the code-section contents emitted by
+ /// BinaryEmitter::emitFunctionBody(). Labels, CFI and debug directives do not
+ /// advance the address and are omitted. If \p RecordAddresses is false, only
+ /// calculate the ending address.
+ uint64_t layoutFunctionBody(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Mirror the code-section contents emitted by BinaryEmitter::emitFunction().
+ /// \p Func must pass shouldEmitFunctionFragment(). If \p RecordAddresses is
+ /// false, only calculate the ending address.
+ uint64_t layoutFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out the fragments in \p Section in emission order and return the first
+ /// address after the section. If \p RecordAddresses is false, do not update
+ /// the function or basic-block address maps.
+ uint64_t layoutSection(const BinaryContext &BC,
+ const SectionPlacement &Section, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out Sections toward increasing addresses and return the first address
+ /// after them. This mirrors allocateAt() in relocation mode and the injected
+ /// section allocation in mapCodeSectionsInPlace() otherwise.
+ uint64_t layoutSectionsForward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Mirror the relocation-mode allocateBefore() helper in
+ /// RewriteInstance::mapCodeSections(). Calculate section sizes while
+ /// allocating before \p DotAddress. Return false if subtraction would
+ /// underflow or alignment would place a section before the start of old
+ /// .text (BC.OldTextSectionAddress); otherwise record the layout and return
+ /// true.
+ bool layoutSectionsBackward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Lay out section-mapped and independently placed functions according to
+ /// the mapping rules for the current relocation mode.
+ void layoutFunctions(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Compute the code layout for the current LongJmp iteration by mirroring
+ /// BinaryEmitter emission and RewriteInstance section mapping.
+ void layout(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Update stub addresses after computing the current layout.
void updateStubGroups();
/// -- Relaxation/stub insertion methods --
@@ -189,7 +269,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Expand the range of the stub in StubBB if necessary
Error relaxStub(BinaryBasicBlock &StubBB, bool &Modified);
- /// Helper to resolve a symbol address according to our tentative layout
+ /// Helper to resolve a symbol address according to our computed layout.
uint64_t getSymbolAddress(const BinaryContext &BC, const MCSymbol *Target,
const BinaryBasicBlock *TgtBB) const;
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 47d463d3f7f6f..19373c5729e45 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -342,6 +342,48 @@ bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {
return false;
}
+bool BinaryContext::isCodeSectionBefore(StringRef A, StringRef B) const {
+ if (A == B)
+ return false;
+
+ // If both A and B have names starting with ".text.cold", then
+ // - if opts::HotFunctionsAtEnd is true, we want order
+ // ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
+ // - if opts::HotFunctionsAtEnd is false, we want order
+ // ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
+ if (A.starts_with(getColdCodeSectionName()) &&
+ B.starts_with(getColdCodeSectionName())) {
+ if (A.size() != B.size())
+ return opts::HotFunctionsAtEnd ? A.size() > B.size()
+ : A.size() < B.size();
+ return opts::HotFunctionsAtEnd ? A > B : A < B;
+ }
+
+ // Place hot text movers before anything else.
+ if (opts::HotText) {
+ if (A == getHotTextMoverSectionName())
+ return true;
+ if (B == getHotTextMoverSectionName())
+ return false;
+ }
+
+ // Depending on opts::HotFunctionsAtEnd, place main and warm sections in
+ // order.
+ if (opts::HotFunctionsAtEnd) {
+ if (B == getMainCodeSectionName())
+ return true;
+ if (A == getMainCodeSectionName())
+ return false;
+ return B == getWarmCodeSectionName();
+ }
+
+ if (A == getMainCodeSectionName())
+ return true;
+ if (B == getMainCodeSectionName())
+ return false;
+ return A == getWarmCodeSectionName();
+}
+
std::unique_ptr<MCObjectWriter>
BinaryContext::createObjectWriter(raw_pwrite_stream &OS) {
return MAB->createObjectWriter(OS);
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 9243b6701b918..36add8cfda079 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -88,10 +88,22 @@ size_t padFunction(std::map<std::string, size_t> &FunctionPadding,
return 0;
}
+size_t breakFunctionSize(const BinaryFunction &Function) {
+ for (const std::string &Name : BreakFunctionNames)
+ if (Function.hasNameRegex(Name))
+ return 2;
+ return 0;
+}
+
+StringRef markFunctionBytes(const BinaryContext &BC) {
+ return MarkFuncs ? BC.MIB->getTrapFillValue() : StringRef();
+}
+
size_t padFunctionBefore(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadBeforeSpec, Function);
}
+
size_t padFunctionAfter(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadSpec, Function);
@@ -286,15 +298,7 @@ void BinaryEmitter::emitFunctions() {
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
FunctionFragment &FF) {
- if (Function.size() == 0 && !Function.hasIslandsInfo())
- return false;
-
- if (Function.getState() == BinaryFunction::State::Empty)
- return false;
-
- // Avoid emitting function without instructions when overwriting the original
- // function in-place. Otherwise, emit the empty function to define the symbol.
- if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ if (!shouldEmitFunctionFragment(BC, Function))
return false;
MCSection *Section =
@@ -387,14 +391,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
"first basic block should never be cold");
// Emit UD2 at the beginning if requested by user.
- if (!opts::BreakFunctionNames.empty()) {
- for (std::string &Name : opts::BreakFunctionNames) {
- if (Function.hasNameRegex(Name)) {
- Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
- break;
- }
- }
- }
+ if (const size_t BreakSize = opts::breakFunctionSize(Function))
+ Streamer.emitIntValue(0x0B0F, BreakSize); // UD2: 0F 0B
// Emit code.
emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
@@ -406,8 +404,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
Streamer.emitFill(Padding, MAI.getTextAlignFillValue());
}
- if (opts::MarkFuncs)
- Streamer.emitBytes(BC.MIB->getTrapFillValue());
+ if (StringRef Marker = opts::markFunctionBytes(BC); !Marker.empty())
+ Streamer.emitBytes(Marker);
// Emit CFI end
if (NeedsFDE)
@@ -1226,6 +1224,22 @@ void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
namespace llvm {
namespace bolt {
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function) {
+ if (Function.size() == 0 && !Function.hasIslandsInfo())
+ return false;
+
+ if (Function.getState() == BinaryFunction::State::Empty)
+ return false;
+
+ // Avoid emitting function without instructions when overwriting the original
+ // function in-place. Otherwise, emit the empty function to define the symbol.
+ if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ return false;
+
+ return true;
+}
+
void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
StringRef OrgSecPrefix) {
BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 30ff9dc4ddc71..d528e110984a7 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -11,12 +11,16 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/LongJmp.h"
+#include "bolt/Core/BinaryEmitter.h"
+#include "bolt/Core/FunctionLayout.h"
#include "bolt/Core/ParallelUtilities.h"
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
+#include <cstdint>
+#include <optional>
#define DEBUG_TYPE "longjmp"
@@ -26,7 +30,6 @@ namespace opts {
extern cl::OptionCategory BoltCategory;
extern cl::OptionCategory BoltOptCategory;
extern cl::opt<bool> UseOldText;
-extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool> GroupStubs("group-stubs",
cl::desc("share stubs across functions"),
@@ -40,12 +43,12 @@ static cl::opt<bool>
static cl::opt<bool> RelaxPLT("relax-plt",
cl::desc("indicate PLT proximity to hot text"),
cl::init(true), cl::cat(BoltOptCategory));
-}
+} // namespace opts
namespace llvm {
namespace bolt {
-constexpr unsigned ColdFragAlign = 16;
+static const Align ColdFragmentAlignment(16);
static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();
@@ -124,6 +127,12 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
Stubs[&Func].insert(StubBB.get());
StubBits[StubBB.get()] = BC.MIB->getUncondBranchEncodingSize();
+ LLVM_DEBUG(
+ dbgs() << "BOLT-DEBUG: LongJmp: creating " << (IsCold ? "cold" : "main")
+ << " stub " << StubSym->getName() << " in " << Func.getPrintName()
+ << " at current layout address 0x" << Twine::utohexstr(AtAddress)
+ << " for " << TgtSym->getName() << '\n');
+
if (IsCold) {
registerInMap(ColdLocalStubs[&Func]);
if (opts::GroupStubs && TgtIsFunc)
@@ -172,10 +181,10 @@ BinaryBasicBlock *LongJmpPass::lookupStubFromGroup(
LLVM_DEBUG({
if (Candidates.size() > 1)
- dbgs() << "Considering stub group with " << Candidates.size()
- << " candidates. DotAddress is " << Twine::utohexstr(DotAddress)
- << ", chosen candidate address is "
- << Twine::utohexstr(Cand->first) << "\n";
+ dbgs() << "BOLT-DEBUG: LongJmp: considering stub group with "
+ << Candidates.size() << " candidates at 0x"
+ << Twine::utohexstr(DotAddress) << "; selected candidate at 0x"
+ << Twine::utohexstr(Cand->first) << '\n';
});
return (PCOffset < MinVal || PCOffset > MaxVal) ? nullptr : Cand->second;
}
@@ -294,7 +303,7 @@ void LongJmpPass::updateStubGroups() {
auto update = [&](StubGroupsTy &StubGroups) {
for (auto &KeyVal : StubGroups) {
for (StubTy &Elem : KeyVal.second)
- Elem.first = BBAddresses[Elem.second];
+ Elem.first = BBAddresses.at(Elem.second);
llvm::sort(KeyVal.second, llvm::less_first());
}
};
@@ -307,169 +316,474 @@ void LongJmpPass::updateStubGroups() {
update(ColdStubGroups);
}
-void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {
- const BinaryContext &BC = Func.getBinaryContext();
- uint64_t HotDot = HotAddresses[&Func];
- uint64_t ColdDot = ColdAddresses[&Func];
- bool Cold = false;
- for (const BinaryBasicBlock *BB : Func.getLayout().blocks()) {
- if (Cold || BB->isCold()) {
- Cold = true;
- BBAddresses[BB] = ColdDot;
- ColdDot += BC.computeCodeSize(BB->begin(), BB->end());
- } else {
- BBAddresses[BB] = HotDot;
- HotDot += BC.computeCodeSize(BB->begin(), BB->end());
- }
+uint64_t LongJmpPass::updateSectionAlignment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t Alignment) const {
+ if (BC.HasRelocations) {
+ // BinaryEmitter::emitFunction() raises every emitted code section to at
+ // least BC.AlignFunctions in relocation mode.
+ Alignment = std::max<uint64_t>(Alignment, BC.AlignFunctions);
+
+ // BinaryEmitter::emitAll() sets the main text section to BC.AlignText.
+ if (Func.getCodeSectionName(FF.getFragmentNum()) ==
+ BC.getMainCodeSectionName())
+ Alignment = std::max<uint64_t>(Alignment, BC.AlignText);
+
+ // BinaryEmitter::emitFunction() emits the mandatory minimum function
+ // alignment first.
+ Alignment = std::max<uint64_t>(Alignment, Func.getMinAlignment());
+
+ // BinaryEmitter::emitFunction() emits the preferred function alignment
+ // only when the corresponding maximum padding is nonzero.
+ const uint16_t MaxAlignBytes = FF.isSplitFragment()
+ ? Func.getMaxColdAlignmentBytes()
+ : Func.getMaxAlignmentBytes();
+ if (MaxAlignBytes > 0)
+ Alignment = std::max<uint64_t>(Alignment, Func.getAlignment());
+ } else {
+ // In non-relocation mode BinaryEmitter emits only the preferred function
+ // alignment. This path is used for newly allocated injected sections.
+ Alignment = std::max<uint64_t>(Alignment, Func.getAlignment());
}
+
+ // BinaryEmitter::emitFunctionBody() emits enabled basic-block alignment
+ // directives.
+ if (BC.AlignBlocks || BC.PreserveBlocksAlignment)
+ for (const BinaryBasicBlock *BB : FF)
+ if (BB->getAlignment() > 1)
+ Alignment = std::max<uint64_t>(Alignment, BB->getAlignment());
+
+ // BinaryEmitter::emitConstantIslands() aligns owned and cloned islands
+ // using the host function's constant-island alignment.
+ if (Func.hasIslandsInfo())
+ Alignment =
+ std::max<uint64_t>(Alignment, Func.getConstantIslandAlignment());
+
+ return Alignment;
}
-uint64_t LongJmpPass::tentativeLayoutRelocColdPart(
- const BinaryContext &BC, BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress) {
- DotAddress =
- alignTo(DotAddress, std::max<uint64_t>(BC.AlignFunctions,
- BC.MaxColdCodeAlignment.load()));
- for (BinaryFunction *Func : SortedFunctions) {
- if (!Func->isSplit())
+void LongJmpPass::assignFunctionFragmentToSection(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF) {
+ const StringRef SectionName = Func.getCodeSectionName(FF.getFragmentNum());
+ auto It = llvm::find_if(Sections, [&](const SectionPlacement &Section) {
+ return StringRef(Section.Name) == SectionName;
+ });
+ if (It == Sections.end()) {
+ // AArch64ELFStreamer::changeSection() gives every text section a
+ // four-byte minimum alignment before BinaryEmitter raises it further.
+ Sections.push_back({SmallString<32>(SectionName),
+ {},
+ updateSectionAlignment(BC, Func, FF, 4)});
+ It = std::prev(Sections.end());
+ } else {
+ It->Alignment = updateSectionAlignment(BC, Func, FF, It->Alignment);
+ }
+
+ It->Fragments.push_back({&Func, FF.getFragmentNum()});
+}
+
+void LongJmpPass::assignFunctionsToSections(
+ const BinaryContext &BC, const BinaryFunctionListType &SortedFunctions) {
+ // Mirror BinaryEmitter::emitFunctions(): emit each main fragment followed
+ // immediately by its split fragments, preserving that order per section.
+ for (const BinaryFunction *Func : SortedFunctions) {
+ // Do not assign functions for which BinaryEmitter::emitFunction()
+ // returns before selecting a section.
+ if (!shouldEmitFunctionFragment(BC, *Func))
+ continue;
+
+ // RewriteInstance ultimately excludes every code section with a
+ // pre-assigned output address. At this point, those are the sections of
+ // fixed-address injected functions.
+ if (Func->isInjected() && Func->getOutputAddress())
+ continue;
+
+ // In relocation mode, process all remaining functions. In non-relocation
+ // mode, process only non-fixed injected functions. Their sections are
+ // allocated after the moved cold fragments and require the same alignment
+ // calculation as relocation-mode sections.
+ if (!BC.HasRelocations && !Func->isInjected())
continue;
- DotAddress = alignTo(DotAddress, Func->getMinAlignment());
- uint64_t Pad =
- offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
- if (Pad <= Func->getMaxColdAlignmentBytes())
- DotAddress += Pad;
- ColdAddresses[Func] = DotAddress;
- LLVM_DEBUG(dbgs() << Func->getPrintName() << " cold tentative: "
- << Twine::utohexstr(DotAddress) << "\n");
- DotAddress += Func->estimateColdSize();
- if (uint64_t IslandSize = Func->estimateConstantIslandSize()) {
- DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
- DotAddress += IslandSize;
+
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: collecting fragments for "
+ << Func->getPrintName() << " (#"
+ << Func->getFunctionNumber() << ")\n");
+
+ const FunctionLayout &Layout = Func->getLayout();
+ assignFunctionFragmentToSection(BC, *Func, Layout.getMainFragment());
+
+ if (Func->isSplit()) {
+ assert(!Func->isInjected() && "injected functions cannot be split");
+ assert((Layout.fragment_size() == 1 || Func->isSimple()) &&
+ "only simple functions can have multiple fragments");
+ for (const FunctionFragment &FF : Layout.getSplitFragments()) {
+ assert(FF.getFragmentNum() == FragmentNum::cold() &&
+ "LongJmp supports only main and cold function fragments");
+ // BinaryEmitter::emitFunctions() skips an empty split fragment unless
+ // the function carries a constant island.
+ if (FF.empty() && !Func->hasConstantIsland())
+ continue;
+ assignFunctionFragmentToSection(BC, *Func, FF);
+ }
}
}
- return DotAddress;
}
-uint64_t
-LongJmpPass::tentativeLayoutRelocMode(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress) {
- // Compute hot cold frontier
- int64_t LastHotIndex = -1u;
- uint32_t CurrentIndex = 0;
- if (opts::HotFunctionsAtEnd) {
- for (BinaryFunction *BF : SortedFunctions) {
- if (BF->hasValidIndex()) {
- LastHotIndex = CurrentIndex;
- break;
- }
+/// Advance \p Offset using the rule from
+/// MCObjectStreamer::emitCodeAlignment(). A zero maximum makes the alignment
+/// mandatory; otherwise omit padding larger than \p MaxBytesToEmit.
+static uint64_t applyCodeAlignment(uint64_t Offset, Align Alignment,
+ uint64_t MaxBytesToEmit = 0) {
+ const uint64_t Pad = offsetToAlignment(Offset, Alignment);
+ return !MaxBytesToEmit || Pad <= MaxBytesToEmit ? Offset + Pad : Offset;
+}
- ++CurrentIndex;
+uint64_t LongJmpPass::layoutFunctionBody(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses) {
+ for (const BinaryBasicBlock *BB : FF) {
+ // Mirror per-basic-block alignment in BinaryEmitter::emitFunctionBody().
+ if ((BC.AlignBlocks || BC.PreserveBlocksAlignment) &&
+ BB->getAlignment() > 1)
+ DotAddress = applyCodeAlignment(DotAddress, BB->getAlign(),
+ BB->getAlignmentMaxBytes());
+
+ if (RecordAddresses) {
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp layout: basic block "
+ << BB->getName() << " in " << Func.getPrintName()
+ << " starts at 0x" << Twine::utohexstr(DotAddress)
+ << '\n');
+ BBAddresses[BB] = DotAddress;
}
- } else {
- for (BinaryFunction *BF : SortedFunctions) {
- if (!BF->hasValidIndex()) {
- LastHotIndex = CurrentIndex;
- break;
- }
- ++CurrentIndex;
- }
+#ifdef EXPENSIVE_CHECKS
+ // computeCodeSize() skips all pseudo-instructions. Calling
+ // computeInstructionSize() directly would not generally help: unless a
+ // pseudo has an explicit size annotation, it also returns zero.
+ // BinaryEmitter handles CFI pseudos separately by emitting unwind
+ // directives that do not advance the code-section address, but passes
+ // every other pseudo to emitInstruction(). No BOLT path is known to place
+ // another pseudo in an emitted basic block; verify that assumption here.
+ for (const MCInst &Instr : *BB)
+ assert((!BC.MIB->isPseudo(Instr) || BC.MIB->isCFI(Instr)) &&
+ "unexpected non-CFI pseudo in emitted function");
+#endif
+ DotAddress += BC.computeCodeSize(BB->begin(), BB->end());
}
- // Hot
- CurrentIndex = 0;
- bool ColdLayoutDone = false;
- auto runColdLayout = [&]() {
- // Mirror the extra hugify alignment inserted by final section allocation
- // after the last non-cold section. Account for it before assigning cold
- // fragment addresses so range checks see the hot-to-cold gap.
- if (opts::Hugify && !BC.HasFixedLoadAddress && !opts::HotFunctionsAtEnd)
- DotAddress = alignTo(DotAddress, BC.AlignText);
- DotAddress = tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
- ColdLayoutDone = true;
- if (opts::HotFunctionsAtEnd)
- DotAddress = alignTo(DotAddress, BC.AlignText);
- };
- for (BinaryFunction *Func : SortedFunctions) {
- if (!BC.shouldEmit(*Func)) {
- HotAddresses[Func] = Func->getAddress();
- continue;
- }
+ // BinaryEmitter::emitFunctionBody() emits constant islands after the
+ // fragment instructions.
+ if (Func.hasIslandsInfo()) {
+ DotAddress = alignTo(DotAddress, Func.getConstantIslandAlignment());
+ DotAddress += Func.estimateConstantIslandSize();
+ }
- if (!ColdLayoutDone && CurrentIndex >= LastHotIndex)
- runColdLayout();
-
- DotAddress = alignTo(DotAddress, Func->getMinAlignment());
- uint64_t Pad =
- offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
- if (Pad <= Func->getMaxAlignmentBytes())
- DotAddress += Pad;
- HotAddresses[Func] = DotAddress;
- LLVM_DEBUG(dbgs() << Func->getPrintName() << " tentative: "
- << Twine::utohexstr(DotAddress) << "\n");
- if (!Func->isSplit())
- DotAddress += Func->estimateSize();
- else
- DotAddress += Func->estimateHotSize();
-
- if (uint64_t IslandSize = Func->estimateConstantIslandSize()) {
- DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
- DotAddress += IslandSize;
- }
- ++CurrentIndex;
+ return DotAddress;
+}
+
+uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses) {
+ assert(shouldEmitFunctionFragment(BC, Func) &&
+ "attempting to lay out a function BinaryEmitter will not emit");
+
+ const FragmentNum Fragment = FF.getFragmentNum();
+ assert((Fragment == FragmentNum::main() || Fragment == FragmentNum::cold()) &&
+ "LongJmp supports only main and cold function fragments");
+ const bool IsCold = Fragment == FragmentNum::cold();
+
+ const bool HasFixedOutputAddress =
+ Func.isInjected() && Func.getOutputAddress();
+ const bool NeedsRelocationAlignment =
+ BC.HasRelocations && !HasFixedOutputAddress;
+ const bool NeedsNonRelocInjectedAlignment =
+ !BC.HasRelocations && Func.isInjected() && !HasFixedOutputAddress;
+ const bool NeedsNonRelocColdAlignment =
+ !BC.HasRelocations && !Func.isInjected() && IsCold;
+
+ // Apply the alignment that affects the fragment's mapped address.
+ // Section-relative fragments mirror BinaryEmitter::emitFunction(); moved
+ // cold fragments mirror mapCodeSectionsInPlace(). Ordinary non-relocation
+ // main fragments and fixed-address injected functions already have exact
+ // addresses.
+ if (NeedsRelocationAlignment) {
+ DotAddress = alignTo(DotAddress, Func.getMinAlignment());
+ const uint16_t MaxAlignmentBytes =
+ IsCold ? Func.getMaxColdAlignmentBytes() : Func.getMaxAlignmentBytes();
+ if (MaxAlignmentBytes > 0)
+ DotAddress =
+ applyCodeAlignment(DotAddress, Func.getAlign(), MaxAlignmentBytes);
+ } else if (NeedsNonRelocInjectedAlignment) {
+ // Newly allocated injected sections retain BinaryEmitter's regular
+ // non-relocation function alignment.
+ DotAddress = alignTo(DotAddress, Func.getAlign());
+ } else if (NeedsNonRelocColdAlignment) {
+ // mapCodeSectionsInPlace() aligns each moved cold fragment to a hard-coded
+ // 16-byte boundary.
+ DotAddress = alignTo(DotAddress, ColdFragmentAlignment);
+ }
+
+ // BinaryEmitter::emitFunction() places --pad-funcs-before after function
+ // alignment and rejects nonzero padding in non-relocation mode.
+ if (BC.HasRelocations)
+ DotAddress += opts::padFunctionBefore(Func);
+
+ // BinaryEmitter::emitFunction() emits the fragment entry symbols here.
+ if (RecordAddresses) {
+ if (!IsCold)
+ HotAddresses[&Func] = DotAddress;
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp layout: "
+ << (IsCold ? "cold" : "main") << " fragment "
+ << Func.getPrintName() << " starts at 0x"
+ << Twine::utohexstr(DotAddress) << '\n');
}
- // Ensure that tentative code layout always runs for cold blocks.
- if (!ColdLayoutDone)
- runColdLayout();
+ // --break-funcs emits UD2 before the function body.
+ DotAddress += opts::breakFunctionSize(Func);
+
+ DotAddress = layoutFunctionBody(BC, Func, FF, DotAddress, RecordAddresses);
+
+ // BinaryEmitter::emitFunction() emits --pad-funcs after the body in both
+ // relocation and non-relocation modes.
+ DotAddress += opts::padFunctionAfter(Func);
+
+ // --mark-funcs emits the target-specific trap marker after the fragment.
+ DotAddress += opts::markFunctionBytes(BC).size();
+
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: " << (IsCold ? "cold" : "main")
+ << " fragment " << Func.getPrintName() << " ends at 0x"
+ << Twine::utohexstr(DotAddress) << '\n';
+ });
+
+ return DotAddress;
+}
+
+uint64_t LongJmpPass::layoutSection(const BinaryContext &BC,
+ const SectionPlacement &Section,
+ uint64_t DotAddress, bool RecordAddresses) {
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: section " << Section.Name
+ << " starts at 0x" << Twine::utohexstr(DotAddress)
+ << ", alignment 0x" << Twine::utohexstr(Section.Alignment) << ", "
+ << Section.Fragments.size() << " fragments\n";
+ });
+
+ for (const FunctionFragmentPlacement &Placement : Section.Fragments) {
+ const BinaryFunction &Func = *Placement.Func;
- // BBs
- for (BinaryFunction *Func : SortedFunctions)
- tentativeBBLayout(*Func);
+ const FunctionFragment &FF =
+ Func.getLayout().getFragment(Placement.Fragment);
+ DotAddress =
+ layoutFunctionFragment(BC, Func, FF, DotAddress, RecordAddresses);
+ }
+
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: section " << Section.Name
+ << " ends at 0x" << Twine::utohexstr(DotAddress) << '\n';
+ });
return DotAddress;
}
-void LongJmpPass::tentativeLayout(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions) {
- uint64_t DotAddress = BC.LayoutStartAddress;
-
- if (!BC.HasRelocations) {
- for (BinaryFunction *Func : SortedFunctions) {
- HotAddresses[Func] = Func->getAddress();
- DotAddress = alignTo(DotAddress, ColdFragAlign);
- ColdAddresses[Func] = DotAddress;
- if (Func->isSplit())
- DotAddress += Func->estimateColdSize();
- tentativeBBLayout(*Func);
+uint64_t LongJmpPass::layoutSectionsForward(const BinaryContext &BC,
+ uint64_t DotAddress) {
+ const bool AdjustMainSection =
+ BC.HasRelocations &&
+ (opts::HotText || (opts::Hugify && !BC.HasFixedLoadAddress));
+ std::optional<uint64_t> MainSectionEnd = std::nullopt;
+
+ // Mirror allocateAt() in RewriteInstance::mapCodeSections().
+ for (const SectionPlacement &Section : Sections) {
+ DotAddress = alignTo(DotAddress, Section.Alignment);
+ DotAddress = layoutSection(BC, Section, DotAddress);
+
+ if (AdjustMainSection &&
+ StringRef(Section.Name) == BC.getMainCodeSectionName()) {
+ if (opts::HotText)
+ MainSectionEnd = DotAddress;
+ // LongJmp supports only main and cold fragments. Mirror the extra
+ // post-main alignment in allocateAt() for --hugify.
+ if (opts::Hugify && !BC.HasFixedLoadAddress)
+ DotAddress = alignTo(DotAddress, Section.Alignment);
}
+ }
- return;
+ // Mirror RewriteInstance::mapCodeSections() padding used to accommodate
+ // hot-text huge-page mapping. LongJmp does not support warm fragments, so the
+ // hot-text end is the end of the main section. RewriteInstance applies this
+ // adjustment only in allocateAt() to advance the next free address;
+ // allocateBefore() starts from a fixed upper boundary and has no
+ // corresponding adjustment.
+ if (MainSectionEnd)
+ DotAddress = std::max(DotAddress, alignTo(*MainSectionEnd, BC.PageAlign));
+
+ return DotAddress;
+}
+
+bool LongJmpPass::layoutSectionsBackward(const BinaryContext &BC,
+ uint64_t DotAddress) {
+ SmallVector<uint64_t, 4> SectionAddresses(Sections.size());
+ // Mirror allocateBefore() in RewriteInstance::mapCodeSections(): assign
+ // section bases in reverse while preserving their sorted output order.
+ for (size_t I = Sections.size(); I > 0; --I) {
+ const SectionPlacement &Section = Sections[I - 1];
+ uint64_t &SectionAddress = SectionAddresses[I - 1];
+ // Match the BinarySection::getOutputSize() consumed by allocateBefore().
+ const uint64_t SectionSize =
+ layoutSection(BC, Section, 0, /*RecordAddresses=*/false);
+ if (SectionSize > DotAddress)
+ return false;
+ DotAddress -= SectionSize;
+ DotAddress = alignDown(DotAddress, Section.Alignment);
+ if (DotAddress < BC.OldTextSectionAddress)
+ return false;
+ SectionAddress = DotAddress;
}
- // Relocation mode
- uint64_t EstimatedTextSize = 0;
- if (opts::UseOldText) {
- EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0);
-
- // Initial padding
- if (EstimatedTextSize <= BC.OldTextSectionSize) {
- DotAddress = BC.OldTextSectionAddress;
- uint64_t Pad = offsetToAlignment(DotAddress, llvm::Align(BC.AlignText));
- if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) {
- DotAddress += Pad;
+ // Contents within every section are still laid out toward higher addresses.
+ for (size_t I = 0; I < Sections.size(); ++I)
+ layoutSection(BC, Sections[I], SectionAddresses[I]);
+
+ return true;
+}
+
+void LongJmpPass::layoutFunctions(
+ const BinaryContext &BC, const BinaryFunctionListType &SortedFunctions) {
+ if (BC.HasRelocations) {
+ // Mirror the old-text allocation choice in
+ // RewriteInstance::mapCodeSections().
+ bool AllocatedAtOldText = false;
+ if (opts::UseOldText) {
+ if (opts::HotFunctionsAtEnd) {
+ AllocatedAtOldText = layoutSectionsBackward(
+ BC, BC.OldTextSectionAddress + BC.OldTextSectionSize);
+ } else {
+ const uint64_t EndAddress =
+ layoutSectionsForward(BC, BC.OldTextSectionAddress);
+ AllocatedAtOldText =
+ EndAddress <= BC.OldTextSectionAddress + BC.OldTextSectionSize;
}
+
+ if (!AllocatedAtOldText) {
+ BC.errs() << "BOLT-WARNING: --use-old-text failed during LongJmp "
+ "layout. The original .text is too small to fit the new "
+ "code.\n";
+ // Do not clear opts::UseOldText here. RewriteInstance also uses it
+ // during emission to decide how to handle non-code sections such as
+ // .eh_frame, and performs the authoritative fallback later.
+ } else {
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: The layout fits into the "
+ "original .text section\n");
+ }
+ }
+
+ // mapCodeSections() falls back to allocateAt() when old text is unused
+ // or too small.
+ if (!AllocatedAtOldText)
+ layoutSectionsForward(BC, BC.LayoutStartAddress);
+ } else {
+ // Mirror RewriteInstance::mapCodeSectionsInPlace(). Main fragments retain
+ // their input addresses, while split cold fragments are appended in
+ // original-function order starting at the first free output address.
+ uint64_t ColdAddress = BC.LayoutStartAddress;
+ for (const auto &BFI : BC.getBinaryFunctions()) {
+ const BinaryFunction &Func = BFI.second;
+
+ // PopulateOutputFunctions excludes functions for which shouldEmit()
+ // returns false. LongJmp never relaxes them, so they need no entries in
+ // the function or basic-block address maps. Unlike the other layout
+ // loops, this one visits getBinaryFunctions() to mirror
+ // mapCodeSectionsInPlace(), and therefore needs an explicit check.
+ if (!BC.shouldEmit(Func))
+ continue;
+
+ if (!shouldEmitFunctionFragment(BC, Func))
+ continue;
+
+ layoutFunctionFragment(BC, Func, Func.getLayout().getMainFragment(),
+ Func.getAddress());
+
+ if (Func.isSplit()) {
+ assert(Func.getLayout().isHotColdSplit() &&
+ "non-relocation mode supports only hot/cold splitting");
+ ColdAddress = layoutFunctionFragment(
+ BC, Func, Func.getLayout().getFragment(FragmentNum::cold()),
+ ColdAddress);
+ }
+ }
+
+ // mapCodeSectionsInPlace() allocates non-fixed injected sections, normally
+ // .text.injected, immediately after the moved cold fragments. These are
+ // the only entries in Sections in non-relocation mode.
+ layoutSectionsForward(BC, ColdAddress);
+ }
+
+ // Fixed-address injected functions are outside Sections and use their
+ // pre-assigned output addresses.
+ for (const BinaryFunction *Func : SortedFunctions) {
+ if (!shouldEmitFunctionFragment(BC, *Func))
+ continue;
+
+ if (Func->isInjected() && Func->getOutputAddress()) {
+ assert(!Func->isSplit() && "injected functions cannot be split");
+ layoutFunctionFragment(BC, *Func, Func->getLayout().getMainFragment(),
+ Func->getOutputAddress());
}
}
+}
- if (!EstimatedTextSize || EstimatedTextSize > BC.OldTextSectionSize) {
- uint64_t TextAlign =
- std::max<uint64_t>(BC.AlignText, BC.MaxMainCodeAlignment.load());
- DotAddress = alignTo(BC.LayoutStartAddress, TextAlign);
+void LongJmpPass::layout(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions) {
+ HotAddresses.clear();
+ BBAddresses.clear();
+ Sections.clear();
+
+ LLVM_DEBUG(
+ dbgs() << "BOLT-DEBUG: LongJmp layout starts at 0x"
+ << Twine::utohexstr(BC.LayoutStartAddress) << ", text alignment 0x"
+ << Twine::utohexstr(BC.AlignText) << ", function alignment 0x"
+ << Twine::utohexstr(BC.AlignFunctions)
+ << ", maximum main alignment 0x"
+ << Twine::utohexstr(BC.MaxMainCodeAlignment.load())
+ << ", maximum cold alignment 0x"
+ << Twine::utohexstr(BC.MaxColdCodeAlignment.load()) << '\n');
+
+ // Reproduce the code placement performed later by BinaryEmitter and
+ // RewriteInstance. First catalogue fragments whose addresses are determined
+ // by output-section placement. In relocation mode this includes all emitted
+ // fragments except fixed-address injected patches. In non-relocation mode it
+ // includes only non-fixed injected functions; ordinary main fragments remain
+ // at their input addresses, while mapCodeSectionsInPlace() allocates moved
+ // cold fragments directly.
+ //
+ // Section alignment depends on every fragment assigned to the section, so
+ // the complete catalogue must be built before calculating any section base.
+ // The layout phase then mirrors either mapCodeSections() or
+ // mapCodeSectionsInPlace(), and finally records fixed injected patches whose
+ // addresses do not come from the section catalogue.
+
+ assignFunctionsToSections(BC, SortedFunctions);
+
+ if (BC.HasRelocations) {
+ // Mirror RewriteInstance::getCodeSections(). Sections not named explicitly
+ // retain their first-emission order.
+ llvm::stable_sort(
+ Sections, [&](const SectionPlacement &A, const SectionPlacement &B) {
+ return BC.isCodeSectionBefore(A.Name, B.Name);
+ });
}
- tentativeLayoutRelocMode(BC, SortedFunctions, DotAddress);
+ layoutFunctions(BC, SortedFunctions);
}
bool LongJmpPass::usesStub(const BinaryFunction &Func,
@@ -520,7 +834,7 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
const MCSymbol *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB.begin());
const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(RealTargetSym);
uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB);
- uint64_t DotAddress = BBAddresses[&StubBB];
+ uint64_t DotAddress = BBAddresses.at(&StubBB);
uint64_t PCRelTgtAddress = DotAddress > TgtAddress ? DotAddress - TgtAddress
: TgtAddress - DotAddress;
@@ -533,10 +847,9 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
if (Bits >= RangeShortJmp)
return Error::success();
- LLVM_DEBUG(dbgs() << "Relaxing stub to short jump. PCRelTgtAddress = "
- << Twine::utohexstr(PCRelTgtAddress)
- << " RealTargetSym = " << RealTargetSym->getName()
- << "\n");
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: relaxing stub to short jump; "
+ << "distance 0x" << Twine::utohexstr(PCRelTgtAddress)
+ << ", target " << RealTargetSym->getName() << '\n');
relaxStubToShortJmp(StubBB, RealTargetSym);
StubBits[&StubBB] = RangeShortJmp;
Modified = true;
@@ -549,9 +862,9 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
return createFatalBOLTError(
"BOLT-ERROR: Unable to relax stub for PIC binary\n");
- LLVM_DEBUG(dbgs() << "Relaxing stub to long jump. PCRelTgtAddress = "
- << Twine::utohexstr(PCRelTgtAddress)
- << " RealTargetSym = " << RealTargetSym->getName() << "\n");
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: relaxing stub to long jump; "
+ << "distance 0x" << Twine::utohexstr(PCRelTgtAddress)
+ << ", target " << RealTargetSym->getName() << '\n');
relaxStubToLongJmp(StubBB, RealTargetSym);
StubBits[&StubBB] = static_cast<int>(BC.AsmInfo->getCodePointerSize() * 8);
Modified = true;
@@ -582,26 +895,41 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,
uint64_t PCRelTgtAddress = getSymbolAddress(BC, TgtSym, TgtBB);
int64_t PCOffset = (int64_t)(PCRelTgtAddress - DotAddress);
- return PCOffset < MinVal || PCOffset > MaxVal;
+ const bool Result = PCOffset < MinVal || PCOffset > MaxVal;
+ LLVM_DEBUG({
+ if (Result)
+ dbgs() << "BOLT-DEBUG: LongJmp: out-of-range branch in "
+ << Func.getPrintName() << ", basic block " << BB.getName()
+ << ", source 0x" << Twine::utohexstr(DotAddress) << ", target "
+ << TgtSym->getName() << " at 0x"
+ << Twine::utohexstr(PCRelTgtAddress) << ", displacement "
+ << PCOffset << ", range [" << MinVal << ", " << MaxVal << "]\n";
+ });
+ return Result;
}
Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) {
const BinaryContext &BC = Func.getBinaryContext();
assert(BC.isAArch64() && "Unsupported arch");
+ // Keep the relaxation traversal consistent with layout(): functions that
+ // BinaryEmitter will not emit have no entries in BBAddresses.
+ if (!shouldEmitFunctionFragment(BC, Func))
+ return Error::success();
+
constexpr int InsnSize = 4; // AArch64
std::vector<std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>>>
Insertions;
BinaryBasicBlock *Frontier = getBBAtHotColdSplitPoint(Func);
- uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;
+ uint64_t FrontierAddress = Frontier ? BBAddresses.at(Frontier) : 0;
if (FrontierAddress)
FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;
// Add necessary stubs for branch targets we know we can't fit in the
// instruction
for (BinaryBasicBlock &BB : Func) {
- uint64_t DotAddress = BBAddresses[&BB];
+ uint64_t DotAddress = BBAddresses.at(&BB);
// Stubs themselves are relaxed on the next loop
if (Stubs[&Func].count(&BB))
continue;
@@ -1386,7 +1714,9 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
do {
++Iterations;
Modified = false;
- tentativeLayout(BC, Sorted);
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: layout iteration " << Iterations
+ << '\n');
+ layout(BC, Sorted);
updateStubGroups();
for (BinaryFunction *Func : Sorted) {
if (auto E = relax(*Func, Modified))
diff --git a/bolt/lib/Passes/ReorderFunctions.cpp b/bolt/lib/Passes/ReorderFunctions.cpp
index f38f502e75ced..c2fb8d10614fc 100644
--- a/bolt/lib/Passes/ReorderFunctions.cpp
+++ b/bolt/lib/Passes/ReorderFunctions.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/ReorderFunctions.h"
+#include "bolt/Core/BinaryEmitter.h"
#include "bolt/Passes/HFSort.h"
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/STLExtras.h"
@@ -29,9 +30,6 @@ extern cl::OptionCategory BoltOptCategory;
extern cl::opt<unsigned> Verbosity;
extern cl::opt<uint32_t> RandomSeed;
-extern size_t padFunctionBefore(const bolt::BinaryFunction &Function);
-extern size_t padFunctionAfter(const bolt::BinaryFunction &Function);
-
extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions(
"reorder-functions",
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index a5de2b5733355..d635af9d28c10 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4367,52 +4367,11 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
if (Section.hasValidSectionID())
CodeSections.emplace_back(&Section);
- auto compareSections = [&](const BinarySection *A, const BinarySection *B) {
- if (A == B)
- return false;
-
- // If both A and B have names starting with ".text.cold", then
- // - if opts::HotFunctionsAtEnd is true, we want order
- // ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
- // - if opts::HotFunctionsAtEnd is false, we want order
- // ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
- if (A->getName().starts_with(BC->getColdCodeSectionName()) &&
- B->getName().starts_with(BC->getColdCodeSectionName())) {
- if (A->getName().size() != B->getName().size())
- return (opts::HotFunctionsAtEnd)
- ? (A->getName().size() > B->getName().size())
- : (A->getName().size() < B->getName().size());
- return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName())
- : (A->getName() < B->getName());
- }
-
- // Place hot text movers before anything else.
- if (opts::HotText) {
- if (A->getName() == BC->getHotTextMoverSectionName())
- return true;
- if (B->getName() == BC->getHotTextMoverSectionName())
- return false;
- }
-
- // Depending on opts::HotFunctionsAtEnd, place main and warm sections in
- // order.
- if (opts::HotFunctionsAtEnd) {
- if (B->getName() == BC->getMainCodeSectionName())
- return true;
- if (A->getName() == BC->getMainCodeSectionName())
- return false;
- return (B->getName() == BC->getWarmCodeSectionName());
- } else {
- if (A->getName() == BC->getMainCodeSectionName())
- return true;
- if (B->getName() == BC->getMainCodeSectionName())
- return false;
- return (A->getName() == BC->getWarmCodeSectionName());
- }
- };
-
// Determine the order of sections.
- llvm::stable_sort(CodeSections, compareSections);
+ llvm::stable_sort(
+ CodeSections, [&](const BinarySection *A, const BinarySection *B) {
+ return BC->isCodeSectionBefore(A->getName(), B->getName());
+ });
#ifndef NDEBUG
// Verify that the order of sections and functions is consistent.
@@ -4577,21 +4536,36 @@ void RewriteInstance::mapCodeSectionsInPlace(
// Processing in non-relocation mode.
uint64_t NewTextSectionStartAddress = NextAvailableAddress;
+ auto MapMainFragment = [&](BinaryFunction &Func, uint64_t OutputAddress) {
+ ErrorOr<BinarySection &> FuncSection = Func.getCodeSection();
+ assert(FuncSection && "cannot find section for function");
+ FuncSection->setOutputAddress(OutputAddress);
+ LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
+ << Twine::utohexstr(FuncSection->getAllocAddress())
+ << " to 0x" << Twine::utohexstr(OutputAddress) << '\n');
+ MapSection(*FuncSection, OutputAddress);
+ Func.setImageAddress(FuncSection->getAllocAddress());
+ Func.setImageSize(FuncSection->getOutputSize());
+ };
+
+ // Map injected patches at their pre-assigned addresses. Injected functions
+ // returned by BC->getInjectedBinaryFunctions() are not contained in
+ // BC->getBinaryFunctions(). Their sections are unique and are removed after
+ // their contents have been copied in place.
+ for (BinaryFunction *Func : BC->getInjectedBinaryFunctions()) {
+ const uint64_t OutputAddress = Func->getOutputAddress();
+ if (!Func->isEmitted() || !OutputAddress)
+ continue;
+
+ MapMainFragment(*Func, OutputAddress);
+ }
+
for (auto &BFI : BC->getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
if (!Function.isEmitted())
continue;
- ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();
- assert(FuncSection && "cannot find section for function");
- FuncSection->setOutputAddress(Function.getAddress());
- LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
- << Twine::utohexstr(FuncSection->getAllocAddress())
- << " to 0x" << Twine::utohexstr(Function.getAddress())
- << '\n');
- MapSection(*FuncSection, Function.getAddress());
- Function.setImageAddress(FuncSection->getAllocAddress());
- Function.setImageSize(FuncSection->getOutputSize());
+ MapMainFragment(Function, Function.getAddress());
assert(Function.getImageSize() <= Function.getMaxSize() &&
"Unexpected large function");
@@ -4644,6 +4618,37 @@ void RewriteInstance::mapCodeSectionsInPlace(
Section.setOutputFileOffset(
getFileOffsetForAddress(NewTextSectionStartAddress));
}
+
+ // Unlike cold fragments, non-fixed injected functions remain in their
+ // emitted code sections. Allocate those sections immediately after the
+ // moved cold fragments so their placement depends only on code layout. The
+ // functions themselves receive their final output addresses later from
+ // linker-resolved symbols in BinaryFunction::updateOutputValues().
+ SmallVector<BinarySection *, 4> InjectedSections;
+ for (const BinaryFunction *Func : BC->getOutputBinaryFunctions()) {
+ if (!(Func->isInjected() && Func->isEmitted() && !Func->getOutputAddress()))
+ continue;
+
+ assert(!Func->isSplit() && "injected functions cannot be split");
+ ErrorOr<BinarySection &> Section = Func->getCodeSection();
+ if (Section && !llvm::is_contained(InjectedSections, &*Section))
+ InjectedSections.push_back(&*Section);
+ }
+
+ for (BinarySection *Section : InjectedSections) {
+ assert(!Section->getOutputAddress() &&
+ "non-fixed injected section already mapped");
+ NextAvailableAddress =
+ alignTo(NextAvailableAddress, Section->getAlignment());
+ LLVM_DEBUG(
+ dbgs() << "BOLT: mapping injected section " << Section->getName()
+ << " at 0x" << Twine::utohexstr(Section->getAllocAddress())
+ << " to 0x" << Twine::utohexstr(NextAvailableAddress) << "\n");
+ MapSection(*Section, NextAvailableAddress);
+ Section->setOutputAddress(NextAvailableAddress);
+ Section->setOutputFileOffset(getFileOffsetForAddress(NextAvailableAddress));
+ NextAvailableAddress += Section->getOutputSize();
+ }
}
void RewriteInstance::mapAllocatableSections(
diff --git a/bolt/test/AArch64/LongJmp-Layout/basic.s b/bolt/test/AArch64/LongJmp-Layout/basic.s
new file mode 100644
index 0000000000000..1eeeb59841c51
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/basic.s
@@ -0,0 +1,106 @@
+## Check that LongJmp's layout matches actual emission for ordinary
+## functions, function and basic-block alignment, explicit padding, and the
+## optional function boundary markers.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.default.bolt --lite=0 \
+# RUN: --debug-only=longjmp > %t.default.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.default.bolt >> %t.default.log
+# RUN: FileCheck %s --check-prefix=DEFAULT < %t.default.log
+# RUN: llvm-bolt %t.exe -o %t.align.bolt --lite=0 \
+# RUN: --align-text=4096 --align-functions=256 \
+# RUN: --align-functions-max-bytes=255 --preserve-blocks-alignment \
+# RUN: --debug-only=longjmp > %t.align.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.align.bolt >> %t.align.log
+# RUN: FileCheck %s --check-prefix=ALIGN < %t.align.log
+# RUN: llvm-bolt %t.exe -o %t.max-align.bolt --lite=0 \
+# RUN: --align-functions=256 --align-functions-max-bytes=1 \
+# RUN: --debug-only=longjmp > %t.max-align.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.max-align.bolt >> %t.max-align.log
+# RUN: FileCheck %s --check-prefix=MAX-ALIGN < %t.max-align.log
+# RUN: llvm-bolt %t.exe -o %t.padding.bolt --lite=0 \
+# RUN: --pad-funcs-before=second:20 --pad-funcs=first:12 \
+# RUN: --break-funcs=first,second --mark-funcs \
+# RUN: --debug-only=longjmp > %t.padding.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.padding.bolt >> %t.padding.log
+# RUN: FileCheck %s --check-prefix=PADDING < %t.padding.log
+# RUN: %clang %cflags %t.o -o %t.noreloc.exe -nostdlib
+# RUN: llvm-bolt %t.noreloc.exe -o %t.noreloc.bolt --lite=0 \
+# RUN: --preserve-blocks-alignment --debug-only=longjmp \
+# RUN: > %t.noreloc.log 2>&1
+# RUN: FileCheck %s --check-prefix=NONRELOC < %t.noreloc.log
+
+# DEFAULT: LongJmp layout: section .text starts at 0x[[TEXT:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: section .text ends at 0x
+# DEFAULT: _start T [[START]]
+# DEFAULT: first T [[FIRST]]
+# DEFAULT: second T [[SECOND]]
+
+# ALIGN: LongJmp layout: section .text starts at 0x[[TEXT:[0-9a-f]+]], alignment 0x1000
+# ALIGN: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# ALIGN: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
+# ALIGN: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# ALIGN: _start T [[START]]
+# ALIGN: first T [[FIRST]]
+# ALIGN: second T [[SECOND]]
+
+# MAX-ALIGN: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# MAX-ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# MAX-ALIGN: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# MAX-ALIGN: _start T [[START]]
+# MAX-ALIGN: first T [[FIRST]]
+# MAX-ALIGN: second T [[SECOND]]
+
+# PADDING: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# PADDING: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# PADDING: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# PADDING: _start T [[START]]
+# PADDING: first T [[FIRST]]
+# PADDING: second T [[SECOND]]
+
+# NONRELOC: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# NONRELOC: BOLT-DEBUG: LongJmp layout starts at 0x
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x[[FIRST]]
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]*[02468ace]0}}
+
+ .text
+ .p2align 6
+ .globl _start
+ .type _start, %function
+_start:
+ bl first
+ bl second
+ ret
+ .size _start, .-_start
+
+ .p2align 6
+ .globl first
+ .type first, %function
+first:
+ cbz x0, .Lfirst_aligned
+ add x0, x0, #1
+ ret
+ .p2align 5
+.Lfirst_aligned:
+ sub x0, x0, #1
+ ret
+ .size first, .-first
+
+ .p2align 7
+ .globl second
+ .type second, %function
+second:
+ add x1, x1, #1
+ ret
+ .size second, .-second
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/constant-island.s b/bolt/test/AArch64/LongJmp-Layout/constant-island.s
new file mode 100644
index 0000000000000..b007849fa1b25
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/constant-island.s
@@ -0,0 +1,61 @@
+## Check layout of an aligned constant island. With splitting, the
+## cold block's reference also exercises duplicated constant-island emission.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.unsplit.bolt --data %t.fdata --lite=0 \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.unsplit.bolt | FileCheck %s \
+# RUN: --check-prefix=UNSPLIT-SECTIONS \
+# RUN: --implicit-check-not='.text.cold'
+# RUN: llvm-bolt %t.exe -o %t.split.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold --debug-only=longjmp 2>&1 \
+# RUN: | FileCheck %s
+# RUN: llvm-readelf -S %t.split.bolt | FileCheck %s \
+# RUN: --check-prefix=SPLIT-SECTIONS
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# UNSPLIT-SECTIONS: .text
+# SPLIT-SECTIONS: .text
+# SPLIT-SECTIONS: .text.cold
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl island_user
+ ret
+ .size _start, .-_start
+
+ .globl island_user
+ .type island_user, %function
+island_user:
+.entry_island_user:
+# FDATA: 1 island_user #.entry_island_user# 10
+ cbz x0, .Lcold
+.hot_island_user:
+# FDATA: 1 island_user #.hot_island_user# 10
+ ret
+.Lcold:
+ adr x1, .Lconstant_island
+ ldr x1, [x1]
+ ret
+ .size island_user, .-island_user
+
+ .p2align 6
+.Lconstant_island:
+ .xword 0x1122334455667788
+ .xword 0x8877665544332211
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/empty-function.s b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
new file mode 100644
index 0000000000000..8591a8410bf1d
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
@@ -0,0 +1,99 @@
+## Check the two kinds of empty output handled by BinaryEmitter and mirrored by
+## LongJmp. A zero-sized input function is not emitted at all. A function that
+## becomes instruction-less is still emitted in relocation mode to define its
+## main symbol, but is omitted when overwriting the original text in place.
+## Structurally empty split fragments never produce symbols.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+# RUN: %t/empty-function.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 \
+# RUN: --pad-funcs-before=empty:64 --pad-funcs=empty:64 \
+# RUN: --debug-only=longjmp > %t.log 2>&1
+# RUN: FileCheck %s --check-prefix=RELOC \
+# RUN: --implicit-check-not="LongJmp layout: main fragment empty" < %t.log
+# RUN: llvm-nm %t.bolt | FileCheck %s --check-prefix=MAIN-SYMBOL
+# RUN: %clang %cflags %t.o -o %t.noreloc.exe -nostdlib
+# RUN: llvm-bolt %t.noreloc.exe -o %t.noreloc.bolt --lite=0 \
+# RUN: --pad-funcs-before=empty:64 --pad-funcs=empty:64 \
+# RUN: --debug-only=longjmp > %t.noreloc.log 2>&1
+# RUN: FileCheck %s --check-prefix=NONRELOC \
+# RUN: --implicit-check-not="LongJmp layout: main fragment empty" \
+# RUN: --implicit-check-not="LongJmp layout: main fragment nop_only" \
+# RUN: < %t.noreloc.log
+
+## --split-strategy=all deliberately leaves gaps in the fragment numbering
+## when the secondary entry is forced back into the main fragment. This creates
+## structurally empty fragments 1 and 2 followed by non-empty fragment 3.
+## Compact code model bypasses the main/cold-only LongJmp layout path, allowing
+## this subcase to check BinaryEmitter empty-fragment handling directly.
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+# RUN: %t/empty-fragments.s -o %t.fragments.o
+# RUN: %clang %cflags %t.fragments.o -o %t.fragments.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.fragments.exe -o %t.fragments.bolt --lite=0 \
+# RUN: --split-functions --split-strategy=all --compact-code-model
+# RUN: llvm-nm %t.fragments.bolt | FileCheck %s --check-prefix=FRAGMENTS \
+# RUN: --implicit-check-not="_start.cold.0" \
+# RUN: --implicit-check-not="_start.cold.1"
+
+# RELOC: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}, alignment 0x{{[0-9a-f]+}}, 3 fragments
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment next starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only ends at 0x
+
+# MAIN-SYMBOL: T nop_only
+
+# NONRELOC: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment next starts at 0x
+
+# FRAGMENTS: t _start.cold.2
+
+#--- empty-function.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl next
+ ret
+ .size _start, .-_start
+
+ .globl next
+ .type next, %function
+next:
+ ret
+ .size next, .-next
+
+ .globl nop_only
+ .type nop_only, %function
+nop_only:
+ nop
+ .size nop_only, .-nop_only
+
+ .globl empty
+ .type empty, %function
+empty:
+ .size empty, .-empty
+
+ .reloc 0, R_AARCH64_NONE
+
+#--- empty-fragments.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ cbz x0, secondary_entry
+ ret
+ .globl secondary_entry
+secondary_entry:
+ add x0, x0, #1
+ b .Llast
+.Llast:
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s b/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
new file mode 100644
index 0000000000000..b8cd051e0673a
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
@@ -0,0 +1,31 @@
+## Check that LongJmp uses the pre-assigned address of an injected patch.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --use-old-text=0 \
+# RUN: --force-patch --debug-only=longjmp > %t.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment patched.org.0/ starts at 0x[[PATCH:[0-9a-f]+]]
+# CHECK: patched.org.0 t [[PATCH]]
+
+ .text
+ .balign 4
+ .globl patched
+ .type patched, %function
+patched:
+ .rept 32
+ nop
+ .endr
+ ret
+ .size patched, .-patched
+
+ .globl _start
+ .type _start, %function
+_start:
+ bl patched
+ ret
+ .size _start, .-_start
diff --git a/bolt/test/AArch64/LongJmp-Layout/function-padding.s b/bolt/test/AArch64/LongJmp-Layout/function-padding.s
new file mode 100644
index 0000000000000..1445eb8ce9de6
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/function-padding.s
@@ -0,0 +1,55 @@
+## Check that LongJmp includes padding before and after functions in its
+## layout. Either padding request puts target outside the range of
+## foo's direct call and requires a stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.after.bolt --data %t.fdata --lite=0 \
+# RUN: --pad-funcs=foo:134217728 2>&1 | FileCheck %s --check-prefix=MAIN
+# RUN: llvm-bolt %t.exe -o %t.before.bolt --data %t.fdata --lite=0 \
+# RUN: --pad-funcs-before=target:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=MAIN
+# RUN: llvm-bolt %t.exe -o %t.split-before.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold \
+# RUN: --pad-funcs-before=target:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SPLIT-BEFORE
+# RUN: llvm-bolt %t.exe -o %t.split-after.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold --pad-funcs=foo:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SPLIT-AFTER
+
+# MAIN: BOLT-INFO: Inserted 1 stubs in the hot area and 0 stubs in the cold area.
+# SPLIT-BEFORE: BOLT-INFO: Inserted 0 stubs in the hot area and 1 stubs in the cold area.
+# SPLIT-AFTER: BOLT-INFO: Inserted 1 stubs in the hot area and 1 stubs in the cold area.
+
+ .text
+ .globl foo
+ .type foo, %function
+foo:
+.entry_foo:
+# FDATA: 1 foo #.entry_foo# 10
+ cbnz x0, .hot_foo
+.Lcold:
+ bl target
+ ret
+.hot_foo:
+# FDATA: 1 foo #.hot_foo# 10
+ ret
+ .size foo, .-foo
+
+ .globl target
+ .type target, %function
+target:
+ ret
+ .size target, .-target
+
+ .globl _start
+ .type _start, %function
+_start:
+ nop
+ ret
+
+ .size _start, .-_start
diff --git a/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s b/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
new file mode 100644
index 0000000000000..79b862d4cc26f
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
@@ -0,0 +1,105 @@
+## Check that LongJmp mirrors code-section placement when cold sections are
+## placed before hot sections with --hot-functions-at-end, including the
+## backward section allocation used by --use-old-text.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.hot.bolt --data %t.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --hot-functions-at-end \
+# RUN: --align-text=16 --align-functions=4 --debug-only=longjmp \
+# RUN: > %t.hot.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.hot.bolt >> %t.hot.log
+# RUN: FileCheck %s --check-prefix=HOT < %t.hot.log
+# RUN: llvm-bolt %t.exe -o %t.old.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --reorder-functions=exec-count \
+# RUN: --hot-functions-at-end --align-text=4 --align-functions=4 \
+# RUN: --debug-only=longjmp > %t.old.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.old.bolt >> %t.old.log
+# RUN: FileCheck %s --check-prefix=OLD < %t.old.log
+# RUN: llvm-bolt %t.exe -o %t.old-forward.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --hot-text=0 --reorder-functions=exec-count \
+# RUN: --align-text=4 --align-functions=4 --debug-only=longjmp \
+# RUN: > %t.old-forward.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.old-forward.bolt >> %t.old-forward.log
+# RUN: FileCheck %s --check-prefix=OLD-FORWARD < %t.old-forward.log
+# RUN: llvm-bolt %t.exe -o %t.old-forward-fail.bolt --data %t.fdata \
+# RUN: --lite=0 --use-old-text --hot-text=0 \
+# RUN: --reorder-functions=exec-count --align-text=4 --align-functions=4 \
+# RUN: --pad-funcs=_start:1024 --debug-only=longjmp 2>&1 | \
+# RUN: FileCheck %s --check-prefix=OLD-FORWARD-FAIL
+# RUN: llvm-bolt %t.exe -o %t.old-fail.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --reorder-functions=exec-count \
+# RUN: --hot-functions-at-end --align-text=4 --align-functions=4 \
+# RUN: --pad-funcs=_start:1024 --debug-only=longjmp 2>&1 | \
+# RUN: FileCheck %s --check-prefix=OLD-FAIL
+
+# HOT: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x{{[0-9a-f]+}}
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[COLD:[0-9a-f]+]]
+# HOT: BOLT-DEBUG: LongJmp layout: section .text.cold ends at 0x
+# HOT: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[HOTFUNC:[0-9a-f]+]]
+# HOT: cold_function T [[COLD]]
+# HOT: _start T [[START]]
+# HOT: hot T [[HOTFUNC]]
+
+# OLD: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x{{[0-9a-f]+}}
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[OLD_COLD:[0-9a-f]+]]
+# OLD: BOLT-DEBUG: LongJmp layout: section .text.cold ends at 0x
+# OLD: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[OLD_START:[0-9a-f]+]]
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[OLD_HOT:[0-9a-f]+]]
+# OLD: BOLT-INFO: using original .text for new code
+# OLD: cold_function T [[OLD_COLD]]
+# OLD: _start T [[OLD_START]]
+# OLD: hot T [[OLD_HOT]]
+
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[FORWARD_START:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[FORWARD_HOT:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[FORWARD_COLD:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-INFO: using original .text for new code
+# OLD-FORWARD: _start T [[FORWARD_START]]
+# OLD-FORWARD: hot T [[FORWARD_HOT]]
+# OLD-FORWARD: cold_function T [[FORWARD_COLD]]
+
+# OLD-FORWARD-FAIL: BOLT-WARNING: --use-old-text failed during LongJmp layout.
+# OLD-FORWARD-FAIL: BOLT-WARNING: --use-old-text failed. The original .text
+# OLD-FORWARD-FAIL-NOT: BOLT-INFO: using original .text for new code
+
+# OLD-FAIL: BOLT-WARNING: --use-old-text failed during LongJmp layout.
+# OLD-FAIL: BOLT-WARNING: --use-old-text failed. The original .text
+# OLD-FAIL-NOT: BOLT-INFO: using original .text for new code
+
+ .text
+ .space 512, 0
+
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+ bl hot
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+.entry_hot:
+ ret
+ .size hot, .-hot
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
+
+# FDATA: 1 _start #.entry_start# 10
+# FDATA: 1 hot #.entry_hot# 10
diff --git a/bolt/test/AArch64/LongJmp-Layout/hugify.s b/bolt/test/AArch64/LongJmp-Layout/hugify.s
new file mode 100644
index 0000000000000..ab9fa02822864
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/hugify.s
@@ -0,0 +1,48 @@
+## Check the extra alignment between the main and cold code sections inserted
+## for --hugify.
+
+# REQUIRES: system-linux, asserts, bolt-runtime, target=aarch64{{.*}}
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t.fdata --lite=0 --hugify \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=HUGIFY-SECTIONS
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# HUGIFY-SECTIONS: .text PROGBITS {{[0-9a-f]+}}00000
+# HUGIFY-SECTIONS: .text.cold PROGBITS {{[0-9a-f]+}}00000
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl hot
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+.entry_hot:
+# FDATA: 1 hot #.entry_hot# 10
+ ret
+ .size hot, .-hot
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ add x0, x0, #1
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s b/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
new file mode 100644
index 0000000000000..c4cf8ce3c25be
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
@@ -0,0 +1,39 @@
+## Check that non-fixed injected code is allocated immediately after split cold
+## fragments in non-relocation mode and that LongJmp uses the same address.
+
+# REQUIRES: system-linux, asserts, bolt-runtime, target=aarch64{{.*}}
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/input.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t/profile.fdata --lite=0 --hugify \
+# RUN: --split-functions --split-all-cold --debug-only=longjmp \
+# RUN: 2>&1 | tee %t.log
+# RUN: llvm-nm --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment _start ends at 0x[[INJECTED:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text.injected starts at 0x[[INJECTED]]
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment __bolt_hugify_start_program starts at 0x[[INJECTED]]
+# CHECK: __bolt_hugify_start_program t [[INJECTED]]
+
+#--- input.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ cbnz x0, .Lhot
+.Lcold:
+ sub x0, x0, #1
+ ret
+.Lhot:
+ add x0, x0, #1
+ ret
+ .size _start, .-_start
+
+#--- profile.fdata
+no_lbr
+1 _start 0 10
+1 _start 4 0
+1 _start c 10
diff --git a/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s b/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
new file mode 100644
index 0000000000000..d9aaf60fef1bb
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
@@ -0,0 +1,33 @@
+## Check that non-simple functions retained at their input addresses are not
+## included in LongJmp's layout in non-relocation mode.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --print-cfg \
+# RUN: --print-only=retained --debug-only=longjmp > %t.log 2>&1
+# RUN: FileCheck %s < %t.log
+
+# CHECK: Binary Function "retained" after building cfg
+# CHECK: IsSimple : 0
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# CHECK-NOT: BOLT-DEBUG: LongJmp layout: main fragment retained
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ ret
+ .size _start, .-_start
+
+## The unknown indirect branch makes the function non-simple without removing
+## its CFG or instructions. In non-relocation mode,
+## shouldEmitFunctionFragment() accepts it, while BinaryContext::shouldEmit()
+## rejects it.
+ .globl retained
+ .type retained, %function
+retained:
+ br x0
+ ret
+ .size retained, .-retained
diff --git a/bolt/test/AArch64/LongJmp-Layout/sections.s b/bolt/test/AArch64/LongJmp-Layout/sections.s
new file mode 100644
index 0000000000000..012cda1bec784
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/sections.s
@@ -0,0 +1,91 @@
+## Check section-aware layout for profiled functions in .text,
+## unprofiled whole functions in .text.cold, and cold basic blocks split from a
+## hot function. The middle cold block is split only with --split-all-cold.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/input.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.unsplit.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --debug-only=longjmp 2>&1 \
+# RUN: | FileCheck %s
+# RUN: llvm-nm -n %t.unsplit.bolt | FileCheck %s --check-prefix=UNSPLIT \
+# RUN: --implicit-check-not='hot.cold.0' \
+# RUN: --implicit-check-not='trailing.cold.0'
+# RUN: llvm-bolt %t.exe -o %t.split.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-nm -n %t.split.bolt | FileCheck %s --check-prefix=SPLIT \
+# RUN: --implicit-check-not='hot.cold.0'
+# RUN: llvm-bolt %t.exe -o %t.split-all.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions --split-all-cold \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-nm -n %t.split-all.bolt | FileCheck %s --check-prefix=SPLIT-ALL
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# UNSPLIT: T hot
+# UNSPLIT: T trailing
+# SPLIT: t trailing.cold.0
+# SPLIT-ALL: t hot.cold.0
+# SPLIT-ALL: t trailing.cold.0
+
+#--- input.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl hot
+ bl trailing
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+ cbnz x0, .Lhot
+.Lcold_middle:
+ sub x0, x0, #1
+ b .Lexit
+.Lhot:
+ add x0, x0, #1
+.Lexit:
+ ret
+ .size hot, .-hot
+
+ .globl trailing
+ .type trailing, %function
+trailing:
+ cbz x1, .Ltrailing_cold
+ add x1, x1, #1
+ ret
+.Ltrailing_cold:
+ sub x1, x1, #1
+ ret
+ .size trailing, .-trailing
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ add x2, x2, #1
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
+
+#--- profile.fdata
+no_lbr
+1 _start 0 10
+1 hot 0 10
+1 hot 4 0
+1 hot c 10
+1 hot 10 10
+1 trailing 0 10
+1 trailing 4 10
+1 trailing c 0
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s b/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
new file mode 100644
index 0000000000000..e8cb41a2a2ba1
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
@@ -0,0 +1,84 @@
+## Check that functions marked ignored after the output function list is built
+## do not affect LongJmp's section layout. Use --force-patch to
+## make this state deterministic. Patching late_ignored fails because the
+## function is too small, so PatchEntries marks it ignored after PopulateOutputFunctions
+## has already included it as a profiled function in the output list.
+##
+## Before the patch, the frontier scan counts late_ignored, but the subsequent
+## layout loop skips it. LongJmp therefore estimates this layout:
+##
+## _start.hot | 128 MiB padding | separator | _start.cold | end
+##
+## The conditional branch from _start.hot to _start.cold appears out of range,
+## so LongJmp inserts a hot stub. The cold call to end appears to be in range,
+## so no cold stub is inserted. The emitter skips late_ignored consistently
+## while placing the fragments, and produces this layout instead:
+##
+## _start.hot | _start.cold | 128 MiB padding | separator | end
+##
+## Here, the conditional branch is in range but the cold call is out of range,
+## causing an emission failure. After the patch, the layout is built
+## from emitted fragments grouped by output section, so late_ignored is excluded.
+## The estimate then matches the emitted layout, and LongJmp inserts the required
+## cold stub instead of the unnecessary hot stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions --split-all-cold \
+# RUN: --skip-funcs=skipped --force-patch \
+# RUN: --pad-funcs-before=separator:134217728 2>&1 \
+# RUN: | FileCheck %s
+
+# CHECK: BOLT-WARNING: failed to patch entries in late_ignored
+# CHECK: BOLT-INFO: Inserted 0 stubs in the hot area and 1 stubs in the cold area.
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl late_ignored
+ cbnz x0, .hot_start
+.cold_start:
+ bl end
+ ret
+.hot_start:
+# FDATA: 1 _start #.hot_start# 10
+ ret
+ .size _start, .-_start
+
+ .globl late_ignored
+ .type late_ignored, %function
+late_ignored:
+.entry_late_ignored:
+# FDATA: 1 late_ignored #.entry_late_ignored# 5
+ ret
+ .size late_ignored, .-late_ignored
+
+ .globl skipped
+ .type skipped, %function
+skipped:
+ ret
+ .size skipped, .-skipped
+
+ .globl separator
+ .type separator, %function
+separator:
+ add x0, x0, #1
+ add x0, x0, #1
+ ret
+ .size separator, .-separator
+
+ .globl end
+ .type end, %function
+end:
+ add x1, x1, #1
+ add x1, x1, #1
+ ret
+ .size end, .-end
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s b/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
new file mode 100644
index 0000000000000..a4478850cd0f8
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
@@ -0,0 +1,44 @@
+## Check that a skipped function and padding requested for it are both absent
+## from LongJmp's emitted-section layout.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --skip-funcs=skipped \
+# RUN: --pad-funcs-before=skipped:64 --pad-funcs=skipped:64 \
+# RUN: --debug-only=longjmp > %t.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment end starts at 0x[[END:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+# CHECK: _start T [[START]]
+# CHECK-NEXT: end T [[END]]
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl end
+ ret
+ .size _start, .-_start
+
+ .globl skipped
+ .type skipped, %function
+skipped:
+ add x0, x0, #1
+ ret
+ .size skipped, .-skipped
+
+ .globl end
+ .type end, %function
+end:
+ add x1, x1, #1
+ ret
+ .size end, .-end
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function.s b/bolt/test/AArch64/LongJmp-Layout/skip-function.s
new file mode 100644
index 0000000000000..321f70f2ec571
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function.s
@@ -0,0 +1,61 @@
+## Check that a skipped function does not contribute to the layout.
+## Padding puts end just within range of _start's call. The intervening target
+## function pushes end out of range unless target is skipped.
+##
+## 134217716 is 0x7fffff4. With target skipped, the displacement is:
+##
+## 0x7fffff4 (padding) + 8 (_start) = 0x7fffffc
+##
+## This is the largest positive displacement encodable by bl. When target is
+## emitted, its 8-byte size increases the displacement to 0x8000004, requiring
+## a stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.relaxed.bolt --lite=0 \
+# RUN: --pad-funcs-before=end:134217716 2>&1 \
+# RUN: | FileCheck %s --check-prefix=RELAXED
+# RUN: llvm-objdump -d --no-show-raw-insn %t.relaxed.bolt \
+# RUN: | FileCheck %s --check-prefix=STUB-DISASM
+# RUN: llvm-bolt %t.exe -o %t.skip.bolt --lite=0 \
+# RUN: --pad-funcs-before=end:134217716 --skip-funcs=target 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SKIP
+# RUN: llvm-objdump -d --no-show-raw-insn %t.skip.bolt \
+# RUN: | FileCheck %s --check-prefix=DIRECT-DISASM
+
+# RELAXED: BOLT-INFO: Inserted 1 stubs in the hot area and 0 stubs in the cold area.
+# SKIP: BOLT-INFO: Inserted 0 stubs in the hot area and 0 stubs in the cold area.
+
+# STUB-DISASM-LABEL: <_start>:
+# STUB-DISASM-NEXT: {{.*}} bl {{.*}} <_start+0x8>
+# STUB-DISASM-NEXT: {{.*}} ret
+# STUB-DISASM-NEXT: {{.*}} adrp x16,
+# STUB-DISASM-NEXT: {{.*}} add x16, x16,
+# STUB-DISASM-NEXT: {{.*}} br x16
+# DIRECT-DISASM-LABEL: <_start>:
+# DIRECT-DISASM-NEXT: {{.*}} bl {{.*}} <end>
+# DIRECT-DISASM-NEXT: {{.*}} ret
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl end
+ ret
+ .size _start, .-_start
+
+ .globl target
+ .type target, %function
+target:
+ add x0, x0, #1
+ ret
+ .size target, .-target
+
+ .globl end
+ .type end, %function
+end:
+ ret
+ .size end, .-end
diff --git a/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s b/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
index 03c35e962e99e..6353e813fadea 100644
--- a/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
+++ b/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
@@ -1,4 +1,4 @@
-# The longjump pass may consider branch targets in range during tentative
+# The longjump pass may consider branch targets in range during the
# layout and decide not to insert stubs for them. Later, final section
# allocation may insert alignment padding after the last non-cold text section
# when hugify is enabled. This moves the following cold section farther away,
diff --git a/bolt/test/AArch64/split-funcs-lite.s b/bolt/test/AArch64/split-funcs-lite.s
index 5f95eea17ae75..1f70fd86ef610 100644
--- a/bolt/test/AArch64/split-funcs-lite.s
+++ b/bolt/test/AArch64/split-funcs-lite.s
@@ -1,4 +1,4 @@
-# This test checks that tentative code layout for cold blocks always runs.
+# This test checks that code layout for cold blocks always runs.
# It commonly happens when using lite mode with split functions.
# REQUIRES: system-linux, asserts
@@ -24,4 +24,4 @@ foo:
## Force relocation mode.
.reloc 0, R_AARCH64_NONE
-# CHECK: foo{{.*}} cold tentative: {{.*}}
+# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment foo{{.*}} starts at 0x
More information about the llvm-commits
mailing list