[llvm] [BOLT] Model emitted code sections in LongJmp layout (PR #218232)
Adam Bzowski via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 12:10:12 PDT 2026
https://github.com/adam-bzowski-arm updated https://github.com/llvm/llvm-project/pull/218232
>From b1bc80e6db153a32961f056fb89249d61608ff2e Mon Sep 17 00:00:00 2001
From: Adam Bzowski <Adam.Bzowski at arm.com>
Date: Sun, 23 Aug 2026 11:29:52 +0000
Subject: [PATCH 1/3] [BOLT] Model emitted code sections in LongJmp layout
Change-Id: I40f0d82d3f24a498e1026ee77bb664885a4e9fdd
---
bolt/include/bolt/Core/BinaryContext.h | 87 +++
bolt/include/bolt/Core/BinaryEmitter.h | 24 +
bolt/include/bolt/Core/BinaryFunction.h | 4 +-
bolt/include/bolt/Passes/LongJmp.h | 132 +++-
bolt/lib/Core/BinaryContext.cpp | 6 +
bolt/lib/Core/BinaryEmitter.cpp | 52 +-
bolt/lib/Passes/LongJmp.cpp | 636 +++++++++++++-----
bolt/lib/Passes/ReorderFunctions.cpp | 4 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 160 ++---
bolt/test/AArch64/LongJmp-Layout/basic.s | 106 +++
.../AArch64/LongJmp-Layout/constant-island.s | 61 ++
.../AArch64/LongJmp-Layout/empty-function.s | 99 +++
.../AArch64/LongJmp-Layout/fixed-injected.s | 31 +
.../AArch64/LongJmp-Layout/function-padding.s | 55 ++
.../LongJmp-Layout/hot-functions-at-end.s | 105 +++
bolt/test/AArch64/LongJmp-Layout/hugify.s | 48 ++
.../AArch64/LongJmp-Layout/injected-nonrel.s | 39 ++
.../LongJmp-Layout/non-simple-nonrel.s | 33 +
bolt/test/AArch64/LongJmp-Layout/sections.s | 91 +++
.../LongJmp-Layout/skip-function-frontier.s | 84 +++
.../LongJmp-Layout/skip-function-padding.s | 44 ++
.../AArch64/LongJmp-Layout/skip-function.s | 61 ++
.../long-jmp-hugify-fixup-out-of-range.s | 2 +-
bolt/test/AArch64/split-funcs-lite.s | 4 +-
24 files changed, 1661 insertions(+), 307 deletions(-)
create mode 100644 bolt/test/AArch64/LongJmp-Layout/basic.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/constant-island.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/empty-function.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/function-padding.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/hugify.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/sections.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
create mode 100644 bolt/test/AArch64/LongJmp-Layout/skip-function.s
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 2ccccde342e7f..5b55ccc4f0f55 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -69,6 +69,7 @@ using namespace object;
namespace bolt {
class BinaryFunction;
+class BinaryContext;
using BinaryFunctionListType = std::vector<BinaryFunction *>;
using ConstBinaryFunctionListType = std::vector<const BinaryFunction *>;
@@ -189,6 +190,92 @@ struct JournalingStreams {
Error createNonFatalBOLTError(const Twine &S);
Error createFatalBOLTError(const Twine &S);
+/// Defines the strict weak ordering for BOLT-produced code sections.
+class CodeSectionOrder {
+public:
+ explicit CodeSectionOrder(const BinaryContext &BC);
+ CodeSectionOrder(StringRef ColdSectionName, StringRef HotTextMoverSectionName,
+ StringRef MainSectionName, StringRef WarmSectionName,
+ bool HotText, bool HotFunctionsAtEnd)
+ : ColdSectionName(ColdSectionName),
+ HotTextMoverSectionName(HotTextMoverSectionName),
+ MainSectionName(MainSectionName), WarmSectionName(WarmSectionName),
+ HotText(HotText), HotFunctionsAtEnd(HotFunctionsAtEnd) {}
+
+ bool operator()(StringRef AName, StringRef BName) const {
+ const SectionKind AKind = getKind(AName);
+ const SectionKind BKind = getKind(BName);
+ const unsigned ARank = getRank(AKind);
+ const unsigned BRank = getRank(BKind);
+ if (ARank != BRank)
+ return ARank < BRank;
+
+ if (AKind == SectionKind::Cold) {
+ if (AName.size() != BName.size())
+ return HotFunctionsAtEnd ? AName.size() > BName.size()
+ : AName.size() < BName.size();
+ if (AName != BName)
+ return HotFunctionsAtEnd ? AName > BName : AName < BName;
+ }
+
+ return false;
+ }
+
+private:
+ enum class SectionKind { Mover, Main, Warm, Cold, Other };
+
+ SectionKind getKind(StringRef Name) const {
+ if (HotText && Name == HotTextMoverSectionName)
+ return SectionKind::Mover;
+ if (Name == MainSectionName)
+ return SectionKind::Main;
+ if (Name == WarmSectionName)
+ return SectionKind::Warm;
+ if (Name.starts_with(ColdSectionName))
+ return SectionKind::Cold;
+ return SectionKind::Other;
+ }
+
+ unsigned getRank(SectionKind Kind) const {
+ if (Kind == SectionKind::Mover)
+ return 0;
+ if (HotFunctionsAtEnd) {
+ switch (Kind) {
+ case SectionKind::Other:
+ return 1;
+ case SectionKind::Cold:
+ return 2;
+ case SectionKind::Warm:
+ return 3;
+ case SectionKind::Main:
+ return 4;
+ case SectionKind::Mover:
+ llvm_unreachable("handled above");
+ }
+ }
+ switch (Kind) {
+ case SectionKind::Main:
+ return 1;
+ case SectionKind::Warm:
+ return 2;
+ case SectionKind::Cold:
+ return 3;
+ case SectionKind::Other:
+ return 4;
+ case SectionKind::Mover:
+ llvm_unreachable("handled above");
+ }
+ llvm_unreachable("unknown section kind");
+ }
+
+ StringRef ColdSectionName;
+ StringRef HotTextMoverSectionName;
+ StringRef MainSectionName;
+ StringRef WarmSectionName;
+ bool HotText;
+ bool HotFunctionsAtEnd;
+};
+
class BinaryContext {
BinaryContext() = delete;
diff --git a/bolt/include/bolt/Core/BinaryEmitter.h b/bolt/include/bolt/Core/BinaryEmitter.h
index 8c3e4b8b64cee..969f492cd1f08 100644
--- a/bolt/include/bolt/Core/BinaryEmitter.h
+++ b/bolt/include/bolt/Core/BinaryEmitter.h
@@ -15,6 +15,7 @@
#define BOLT_CORE_BINARY_EMITTER_H
#include "llvm/ADT/StringRef.h"
+#include <cstddef>
namespace llvm {
class MCStreamer;
@@ -24,6 +25,11 @@ class BinaryContext;
class BinaryFunction;
class FunctionFragment;
+/// Return whether BinaryEmitter::emitFunction() proceeds to select an output
+/// section for \p Function.
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function);
+
/// Emit all code and data from the BinaryContext \p BC into the \p Streamer.
///
/// \p OrgSecPrefix is used to modify name of emitted original sections
@@ -40,4 +46,22 @@ void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
} // namespace bolt
} // namespace llvm
+namespace opts {
+
+/// Return the number of bytes emitted before each fragment of \p Function by
+/// --break-funcs.
+std::size_t breakFunctionSize(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the bytes emitted after each fragment by --mark-funcs.
+llvm::StringRef
+markFunctionBytes(const llvm::bolt::BinaryContext &BinaryContext);
+
+/// Return the padding requested before each emitted fragment of \p Function.
+std::size_t padFunctionBefore(const llvm::bolt::BinaryFunction &Function);
+
+/// Return the padding requested after each emitted fragment of \p Function.
+std::size_t padFunctionAfter(const llvm::bolt::BinaryFunction &Function);
+
+} // namespace opts
+
#endif
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 14d7f9b5b5359..6e24dd73ac37b 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -2314,7 +2314,9 @@ class BinaryFunction {
if (!OnBehalfOf) {
for (BinaryFunction *ExternalFunc : Islands->Dependency) {
- Size = alignTo(Size, ExternalFunc->getConstantIslandAlignment());
+ // BinaryEmitter aligns dependency islands using the constant-island
+ // alignment of the host function.
+ Size = alignTo(Size, getConstantIslandAlignment());
Size += ExternalFunc->estimateConstantIslandSize(this);
}
}
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index fa8cf67a95d73..d0eaace481695 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -21,15 +21,15 @@ class BranchLivenessInfo;
/// pull this pass inside BOLT because here we can do a better job at stub
/// inserting by manipulating the CFG, something linkers can't do.
///
-/// We iteratively repeat the following until no modification is done: we do a
-/// tentative layout with the current function sizes; then we add stubs for
+/// We iteratively repeat the following until no modification is done: we
+/// compute the layout with the current function sizes; then we add stubs for
/// branches that we know are out of range or we expand smaller stubs (28-bit)
/// to a large one if necessary (32 or 64).
///
-/// This expansion inserts the equivalent of "linker stubs", small
-/// blocks of code that load a 64-bit address into a pre-allocated register and
-// then executes an unconditional indirect branch on this register. By using a
-/// 64-bit range, we guarantee it can reach any code location.
+/// This expansion inserts the equivalent of "linker stubs": small blocks of
+/// code that load a 64-bit address into a pre-allocated register and then
+/// execute an unconditional indirect branch through that register. By using a
+/// 64-bit range, we guarantee that they can reach any code location.
///
class LongJmpPass : public BinaryFunctionPass {
/// Used to implement stub grouping (reusing a stub from one function into
@@ -52,9 +52,10 @@ class LongJmpPass : public BinaryFunctionPass {
DenseMap<const BinaryFunction *, std::set<const BinaryBasicBlock *>> Stubs;
using FuncAddressesMapTy = DenseMap<const BinaryFunction *, uint64_t>;
- /// Hold tentative addresses
+ /// Main-fragment start addresses for the current layout iteration.
FuncAddressesMapTy HotAddresses;
- FuncAddressesMapTy ColdAddresses;
+
+ /// Basic-block start addresses for the current layout iteration.
DenseMap<const BinaryBasicBlock *, uint64_t> BBAddresses;
/// Used to identify the stub size
@@ -121,23 +122,102 @@ class LongJmpPass : public BinaryFunctionPass {
/// Relax calls using function cluster approach.
void relaxCalls(BinaryContext &BC);
- /// -- Layout estimation methods --
- /// Try to do layout before running the emitter, by looking at BinaryFunctions
- /// and MCInsts -- this is an estimation. To be correct for longjmp inserter
- /// purposes, we need to do a size worst-case estimation. Real layout is done
- /// by RewriteInstance::mapFileSections()
- void tentativeLayout(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions);
- uint64_t tentativeLayoutRelocMode(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- uint64_t tentativeLayoutRelocColdPart(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress);
- void tentativeBBLayout(const BinaryFunction &Func);
-
- /// Update stubs addresses with their exact address after a round of stub
- /// insertion and layout estimation is done.
+ /// Identifies a function fragment and its owning function.
+ struct FunctionFragmentPlacement {
+ /// Function that owns the fragment.
+ const BinaryFunction *Func;
+
+ /// Number identifying the fragment within the function layout.
+ FragmentNum Fragment;
+ };
+
+ /// Describes the placement of one emitted code section.
+ struct SectionPlacement {
+ /// Output section name returned by BinaryFunction::getCodeSectionName().
+ SmallString<32> Name;
+
+ /// Fragments in BinaryEmitter::emitFunctions() emission order.
+ SmallVector<FunctionFragmentPlacement, 0> Fragments;
+
+ /// Maximum alignment required by the section and its contents.
+ uint64_t Alignment;
+ };
+
+ /// Code sections whose addresses are determined during final mapping, and
+ /// their fragments. In relocation mode, sections are sorted like
+ /// RewriteInstance::getCodeSections() before allocation. In non-relocation
+ /// mode, these are the non-fixed injected sections in emission order.
+ SmallVector<SectionPlacement, 4> Sections;
+
+ /// Update \p CurrentAlignment with the requirements added while
+ /// BinaryEmitter emits \p FF. The non-relocation path is used only for
+ /// non-fixed injected sections.
+ uint64_t updateSectionAlignment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t CurrentAlignment) const;
+
+ /// Assign \p FF to its output section, creating its placement if needed.
+ void assignFunctionFragmentToSection(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF);
+
+ /// Assign fragments to output sections in BinaryEmitter::emitFunctions()
+ /// order. In relocation mode, collect all emitted non-fixed fragments. In
+ /// non-relocation mode, collect only non-fixed injected functions. Functions
+ /// mapped independently by RewriteInstance are excluded.
+ void assignFunctionsToSections(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Mirror the code-section contents emitted by
+ /// BinaryEmitter::emitFunctionBody(). Labels, CFI and debug directives do not
+ /// advance the address and are omitted. If \p RecordAddresses is false, only
+ /// calculate the ending address.
+ uint64_t layoutFunctionBody(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Mirror the code-section contents emitted by BinaryEmitter::emitFunction().
+ /// \p Func must pass shouldEmitFunctionFragment(). If \p RecordAddresses is
+ /// false, only calculate the ending address.
+ uint64_t layoutFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out the fragments in \p Section in emission order and return the first
+ /// address after the section. If \p RecordAddresses is false, do not update
+ /// the function or basic-block address maps.
+ uint64_t layoutSection(const BinaryContext &BC,
+ const SectionPlacement &Section, uint64_t DotAddress,
+ bool RecordAddresses = true);
+
+ /// Lay out Sections toward increasing addresses and return the first address
+ /// after them. This mirrors allocateAt() in relocation mode and the injected
+ /// section allocation in mapCodeSectionsInPlace() otherwise.
+ uint64_t layoutSectionsForward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Mirror the relocation-mode allocateBefore() helper in
+ /// RewriteInstance::mapCodeSections(). Calculate section sizes while
+ /// allocating before \p DotAddress. Return false if subtraction would
+ /// underflow or alignment would place a section before the start of old
+ /// .text (BC.OldTextSectionAddress); otherwise record the layout and return
+ /// true.
+ bool layoutSectionsBackward(const BinaryContext &BC, uint64_t DotAddress);
+
+ /// Lay out section-mapped and independently placed functions according to
+ /// the mapping rules for the current relocation mode.
+ void layoutFunctions(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Compute the code layout for the current LongJmp iteration by mirroring
+ /// BinaryEmitter emission and RewriteInstance section mapping.
+ void layout(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions);
+
+ /// Update stub addresses after computing the current layout.
void updateStubGroups();
/// -- Relaxation/stub insertion methods --
@@ -189,7 +269,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Expand the range of the stub in StubBB if necessary
Error relaxStub(BinaryBasicBlock &StubBB, bool &Modified);
- /// Helper to resolve a symbol address according to our tentative layout
+ /// Helper to resolve a symbol address according to our computed layout.
uint64_t getSymbolAddress(const BinaryContext &BC, const MCSymbol *Target,
const BinaryBasicBlock *TgtBB) const;
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index c40851d9aaa98..fac31bb125f92 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -107,6 +107,12 @@ static cl::opt<bool> DropDWOPageCache(
namespace llvm {
namespace bolt {
+CodeSectionOrder::CodeSectionOrder(const BinaryContext &BC)
+ : CodeSectionOrder(
+ BC.getColdCodeSectionName(), BC.getHotTextMoverSectionName(),
+ BC.getMainCodeSectionName(), BC.getWarmCodeSectionName(), opts::HotText,
+ opts::HotFunctionsAtEnd) {}
+
char BOLTError::ID = 0;
BOLTError::BOLTError(bool IsFatal, const Twine &S)
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 29dbcab7945ec..f2289f98a5538 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -88,10 +88,22 @@ size_t padFunction(std::map<std::string, size_t> &FunctionPadding,
return 0;
}
+size_t breakFunctionSize(const BinaryFunction &Function) {
+ for (const std::string &Name : BreakFunctionNames)
+ if (Function.hasNameRegex(Name))
+ return 2;
+ return 0;
+}
+
+StringRef markFunctionBytes(const BinaryContext &BC) {
+ return MarkFuncs ? BC.MIB->getTrapFillValue() : StringRef();
+}
+
size_t padFunctionBefore(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadBeforeSpec, Function);
}
+
size_t padFunctionAfter(const BinaryFunction &Function) {
static std::map<std::string, size_t> CacheFunctionPadding;
return padFunction(CacheFunctionPadding, FunctionPadSpec, Function);
@@ -286,15 +298,7 @@ void BinaryEmitter::emitFunctions() {
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
FunctionFragment &FF) {
- if (Function.size() == 0 && !Function.hasIslandsInfo())
- return false;
-
- if (Function.getState() == BinaryFunction::State::Empty)
- return false;
-
- // Avoid emitting function without instructions when overwriting the original
- // function in-place. Otherwise, emit the empty function to define the symbol.
- if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ if (!shouldEmitFunctionFragment(BC, Function))
return false;
MCSection *Section =
@@ -387,14 +391,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
"first basic block should never be cold");
// Emit UD2 at the beginning if requested by user.
- if (!opts::BreakFunctionNames.empty()) {
- for (std::string &Name : opts::BreakFunctionNames) {
- if (Function.hasNameRegex(Name)) {
- Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
- break;
- }
- }
- }
+ if (const size_t BreakSize = opts::breakFunctionSize(Function))
+ Streamer.emitIntValue(0x0B0F, BreakSize); // UD2: 0F 0B
// Emit code.
emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
@@ -406,8 +404,8 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
Streamer.emitFill(Padding, MAI.getTextAlignFillValue());
}
- if (opts::MarkFuncs)
- Streamer.emitBytes(BC.MIB->getTrapFillValue());
+ if (StringRef Marker = opts::markFunctionBytes(BC); !Marker.empty())
+ Streamer.emitBytes(Marker);
// Emit CFI end
if (NeedsFDE)
@@ -1226,6 +1224,22 @@ void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
namespace llvm {
namespace bolt {
+bool shouldEmitFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Function) {
+ if (Function.size() == 0 && !Function.hasIslandsInfo())
+ return false;
+
+ if (Function.getState() == BinaryFunction::State::Empty)
+ return false;
+
+ // Avoid emitting function without instructions when overwriting the original
+ // function in-place. Otherwise, emit the empty function to define the symbol.
+ if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())
+ return false;
+
+ return true;
+}
+
void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
StringRef OrgSecPrefix) {
BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 30ff9dc4ddc71..3f383f4a83d82 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -11,12 +11,16 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/LongJmp.h"
+#include "bolt/Core/BinaryEmitter.h"
+#include "bolt/Core/FunctionLayout.h"
#include "bolt/Core/ParallelUtilities.h"
#include "bolt/Passes/BranchLivenessUtils.h"
#include "bolt/Passes/RegAnalysis.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/MathExtras.h"
+#include <cstdint>
+#include <optional>
#define DEBUG_TYPE "longjmp"
@@ -26,7 +30,6 @@ namespace opts {
extern cl::OptionCategory BoltCategory;
extern cl::OptionCategory BoltOptCategory;
extern cl::opt<bool> UseOldText;
-extern cl::opt<bool> HotFunctionsAtEnd;
static cl::opt<bool> GroupStubs("group-stubs",
cl::desc("share stubs across functions"),
@@ -40,12 +43,12 @@ static cl::opt<bool>
static cl::opt<bool> RelaxPLT("relax-plt",
cl::desc("indicate PLT proximity to hot text"),
cl::init(true), cl::cat(BoltOptCategory));
-}
+} // namespace opts
namespace llvm {
namespace bolt {
-constexpr unsigned ColdFragAlign = 16;
+static const Align ColdFragmentAlignment(16);
static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();
@@ -124,6 +127,12 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
Stubs[&Func].insert(StubBB.get());
StubBits[StubBB.get()] = BC.MIB->getUncondBranchEncodingSize();
+ LLVM_DEBUG(
+ dbgs() << "BOLT-DEBUG: LongJmp: creating " << (IsCold ? "cold" : "main")
+ << " stub " << StubSym->getName() << " in " << Func.getPrintName()
+ << " at current layout address 0x" << Twine::utohexstr(AtAddress)
+ << " for " << TgtSym->getName() << '\n');
+
if (IsCold) {
registerInMap(ColdLocalStubs[&Func]);
if (opts::GroupStubs && TgtIsFunc)
@@ -172,10 +181,10 @@ BinaryBasicBlock *LongJmpPass::lookupStubFromGroup(
LLVM_DEBUG({
if (Candidates.size() > 1)
- dbgs() << "Considering stub group with " << Candidates.size()
- << " candidates. DotAddress is " << Twine::utohexstr(DotAddress)
- << ", chosen candidate address is "
- << Twine::utohexstr(Cand->first) << "\n";
+ dbgs() << "BOLT-DEBUG: LongJmp: considering stub group with "
+ << Candidates.size() << " candidates at 0x"
+ << Twine::utohexstr(DotAddress) << "; selected candidate at 0x"
+ << Twine::utohexstr(Cand->first) << '\n';
});
return (PCOffset < MinVal || PCOffset > MaxVal) ? nullptr : Cand->second;
}
@@ -294,7 +303,7 @@ void LongJmpPass::updateStubGroups() {
auto update = [&](StubGroupsTy &StubGroups) {
for (auto &KeyVal : StubGroups) {
for (StubTy &Elem : KeyVal.second)
- Elem.first = BBAddresses[Elem.second];
+ Elem.first = BBAddresses.at(Elem.second);
llvm::sort(KeyVal.second, llvm::less_first());
}
};
@@ -307,169 +316,476 @@ void LongJmpPass::updateStubGroups() {
update(ColdStubGroups);
}
-void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {
- const BinaryContext &BC = Func.getBinaryContext();
- uint64_t HotDot = HotAddresses[&Func];
- uint64_t ColdDot = ColdAddresses[&Func];
- bool Cold = false;
- for (const BinaryBasicBlock *BB : Func.getLayout().blocks()) {
- if (Cold || BB->isCold()) {
- Cold = true;
- BBAddresses[BB] = ColdDot;
- ColdDot += BC.computeCodeSize(BB->begin(), BB->end());
- } else {
- BBAddresses[BB] = HotDot;
- HotDot += BC.computeCodeSize(BB->begin(), BB->end());
- }
+uint64_t LongJmpPass::updateSectionAlignment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t Alignment) const {
+ if (BC.HasRelocations) {
+ // BinaryEmitter::emitFunction() raises every emitted code section to at
+ // least BC.AlignFunctions in relocation mode.
+ Alignment = std::max<uint64_t>(Alignment, BC.AlignFunctions);
+
+ // BinaryEmitter::emitAll() sets the main text section to BC.AlignText.
+ if (Func.getCodeSectionName(FF.getFragmentNum()) ==
+ BC.getMainCodeSectionName())
+ Alignment = std::max<uint64_t>(Alignment, BC.AlignText);
+
+ // BinaryEmitter::emitFunction() emits the mandatory minimum function
+ // alignment first.
+ Alignment = std::max<uint64_t>(Alignment, Func.getMinAlignment());
+
+ // BinaryEmitter::emitFunction() emits the preferred function alignment
+ // only when the corresponding maximum padding is nonzero.
+ const uint16_t MaxAlignBytes = FF.isSplitFragment()
+ ? Func.getMaxColdAlignmentBytes()
+ : Func.getMaxAlignmentBytes();
+ if (MaxAlignBytes > 0)
+ Alignment = std::max<uint64_t>(Alignment, Func.getAlignment());
+ } else {
+ // In non-relocation mode BinaryEmitter emits only the preferred function
+ // alignment. This path is used for newly allocated injected sections.
+ Alignment = std::max<uint64_t>(Alignment, Func.getAlignment());
}
+
+ // BinaryEmitter::emitFunctionBody() emits enabled basic-block alignment
+ // directives.
+ if (BC.AlignBlocks || BC.PreserveBlocksAlignment)
+ for (const BinaryBasicBlock *BB : FF)
+ if (BB->getAlignment() > 1)
+ Alignment = std::max<uint64_t>(Alignment, BB->getAlignment());
+
+ // BinaryEmitter::emitConstantIslands() aligns owned and cloned islands
+ // using the host function's constant-island alignment.
+ if (Func.hasIslandsInfo())
+ Alignment =
+ std::max<uint64_t>(Alignment, Func.getConstantIslandAlignment());
+
+ return Alignment;
}
-uint64_t LongJmpPass::tentativeLayoutRelocColdPart(
- const BinaryContext &BC, BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress) {
- DotAddress =
- alignTo(DotAddress, std::max<uint64_t>(BC.AlignFunctions,
- BC.MaxColdCodeAlignment.load()));
- for (BinaryFunction *Func : SortedFunctions) {
- if (!Func->isSplit())
+void LongJmpPass::assignFunctionFragmentToSection(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF) {
+ const StringRef SectionName = Func.getCodeSectionName(FF.getFragmentNum());
+ auto It = llvm::find_if(Sections, [&](const SectionPlacement &Section) {
+ return StringRef(Section.Name) == SectionName;
+ });
+ if (It == Sections.end()) {
+ // AArch64ELFStreamer::changeSection() gives every text section a
+ // four-byte minimum alignment before BinaryEmitter raises it further.
+ Sections.push_back({SmallString<32>(SectionName),
+ {},
+ updateSectionAlignment(BC, Func, FF, 4)});
+ It = std::prev(Sections.end());
+ } else {
+ It->Alignment = updateSectionAlignment(BC, Func, FF, It->Alignment);
+ }
+
+ It->Fragments.push_back({&Func, FF.getFragmentNum()});
+}
+
+void LongJmpPass::assignFunctionsToSections(
+ const BinaryContext &BC, const BinaryFunctionListType &SortedFunctions) {
+ // Mirror BinaryEmitter::emitFunctions(): emit each main fragment followed
+ // immediately by its split fragments, preserving that order per section.
+ for (const BinaryFunction *Func : SortedFunctions) {
+ // Do not assign functions for which BinaryEmitter::emitFunction()
+ // returns before selecting a section.
+ if (!shouldEmitFunctionFragment(BC, *Func))
+ continue;
+
+ // RewriteInstance ultimately excludes every code section with a
+ // pre-assigned output address. At this point, those are the sections of
+ // fixed-address injected functions.
+ if (Func->isInjected() && Func->getOutputAddress())
+ continue;
+
+ // In relocation mode, process all remaining functions. In non-relocation
+ // mode, process only non-fixed injected functions. Their sections are
+ // allocated after the moved cold fragments and require the same alignment
+ // calculation as relocation-mode sections.
+ if (!BC.HasRelocations && !Func->isInjected())
continue;
- DotAddress = alignTo(DotAddress, Func->getMinAlignment());
- uint64_t Pad =
- offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
- if (Pad <= Func->getMaxColdAlignmentBytes())
- DotAddress += Pad;
- ColdAddresses[Func] = DotAddress;
- LLVM_DEBUG(dbgs() << Func->getPrintName() << " cold tentative: "
- << Twine::utohexstr(DotAddress) << "\n");
- DotAddress += Func->estimateColdSize();
- if (uint64_t IslandSize = Func->estimateConstantIslandSize()) {
- DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
- DotAddress += IslandSize;
+
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: collecting fragments for "
+ << Func->getPrintName() << " (#"
+ << Func->getFunctionNumber() << ")\n");
+
+ const FunctionLayout &Layout = Func->getLayout();
+ assignFunctionFragmentToSection(BC, *Func, Layout.getMainFragment());
+
+ if (Func->isSplit()) {
+ assert(!Func->isInjected() && "injected functions cannot be split");
+ assert((Layout.fragment_size() == 1 || Func->isSimple()) &&
+ "only simple functions can have multiple fragments");
+ for (const FunctionFragment &FF : Layout.getSplitFragments()) {
+ assert(FF.getFragmentNum() == FragmentNum::cold() &&
+ "LongJmp supports only main and cold function fragments");
+ // BinaryEmitter::emitFunctions() skips an empty split fragment unless
+ // the function carries a constant island.
+ if (FF.empty() && !Func->hasConstantIsland())
+ continue;
+ assignFunctionFragmentToSection(BC, *Func, FF);
+ }
}
}
- return DotAddress;
}
-uint64_t
-LongJmpPass::tentativeLayoutRelocMode(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions,
- uint64_t DotAddress) {
- // Compute hot cold frontier
- int64_t LastHotIndex = -1u;
- uint32_t CurrentIndex = 0;
- if (opts::HotFunctionsAtEnd) {
- for (BinaryFunction *BF : SortedFunctions) {
- if (BF->hasValidIndex()) {
- LastHotIndex = CurrentIndex;
- break;
- }
+/// Advance \p Offset using the rule from
+/// MCObjectStreamer::emitCodeAlignment(). A zero maximum makes the alignment
+/// mandatory; otherwise omit padding larger than \p MaxBytesToEmit.
+static uint64_t applyCodeAlignment(uint64_t Offset, Align Alignment,
+ uint64_t MaxBytesToEmit = 0) {
+ const uint64_t Pad = offsetToAlignment(Offset, Alignment);
+ return !MaxBytesToEmit || Pad <= MaxBytesToEmit ? Offset + Pad : Offset;
+}
- ++CurrentIndex;
+uint64_t LongJmpPass::layoutFunctionBody(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses) {
+ for (const BinaryBasicBlock *BB : FF) {
+ // Mirror per-basic-block alignment in BinaryEmitter::emitFunctionBody().
+ if ((BC.AlignBlocks || BC.PreserveBlocksAlignment) &&
+ BB->getAlignment() > 1)
+ DotAddress = applyCodeAlignment(DotAddress, BB->getAlign(),
+ BB->getAlignmentMaxBytes());
+
+ if (RecordAddresses) {
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp layout: basic block "
+ << BB->getName() << " in " << Func.getPrintName()
+ << " starts at 0x" << Twine::utohexstr(DotAddress)
+ << '\n');
+ BBAddresses[BB] = DotAddress;
}
- } else {
- for (BinaryFunction *BF : SortedFunctions) {
- if (!BF->hasValidIndex()) {
- LastHotIndex = CurrentIndex;
- break;
- }
- ++CurrentIndex;
- }
+#ifdef EXPENSIVE_CHECKS
+ // computeCodeSize() skips all pseudo-instructions. Calling
+ // computeInstructionSize() directly would not generally help: unless a
+ // pseudo has an explicit size annotation, it also returns zero.
+ // BinaryEmitter handles CFI pseudos separately by emitting unwind
+ // directives that do not advance the code-section address, but passes
+ // every other pseudo to emitInstruction(). No BOLT path is known to place
+ // another pseudo in an emitted basic block; verify that assumption here.
+ for (const MCInst &Instr : *BB)
+ assert((!BC.MIB->isPseudo(Instr) || BC.MIB->isCFI(Instr)) &&
+ "unexpected non-CFI pseudo in emitted function");
+#endif
+ DotAddress += BC.computeCodeSize(BB->begin(), BB->end());
}
- // Hot
- CurrentIndex = 0;
- bool ColdLayoutDone = false;
- auto runColdLayout = [&]() {
- // Mirror the extra hugify alignment inserted by final section allocation
- // after the last non-cold section. Account for it before assigning cold
- // fragment addresses so range checks see the hot-to-cold gap.
- if (opts::Hugify && !BC.HasFixedLoadAddress && !opts::HotFunctionsAtEnd)
- DotAddress = alignTo(DotAddress, BC.AlignText);
- DotAddress = tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
- ColdLayoutDone = true;
- if (opts::HotFunctionsAtEnd)
- DotAddress = alignTo(DotAddress, BC.AlignText);
- };
- for (BinaryFunction *Func : SortedFunctions) {
- if (!BC.shouldEmit(*Func)) {
- HotAddresses[Func] = Func->getAddress();
- continue;
- }
+ // BinaryEmitter::emitFunctionBody() emits constant islands after the
+ // fragment instructions.
+ if (Func.hasIslandsInfo()) {
+ DotAddress = alignTo(DotAddress, Func.getConstantIslandAlignment());
+ DotAddress += Func.estimateConstantIslandSize();
+ }
- if (!ColdLayoutDone && CurrentIndex >= LastHotIndex)
- runColdLayout();
-
- DotAddress = alignTo(DotAddress, Func->getMinAlignment());
- uint64_t Pad =
- offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
- if (Pad <= Func->getMaxAlignmentBytes())
- DotAddress += Pad;
- HotAddresses[Func] = DotAddress;
- LLVM_DEBUG(dbgs() << Func->getPrintName() << " tentative: "
- << Twine::utohexstr(DotAddress) << "\n");
- if (!Func->isSplit())
- DotAddress += Func->estimateSize();
- else
- DotAddress += Func->estimateHotSize();
-
- if (uint64_t IslandSize = Func->estimateConstantIslandSize()) {
- DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
- DotAddress += IslandSize;
- }
- ++CurrentIndex;
+ return DotAddress;
+}
+
+uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
+ const BinaryFunction &Func,
+ const FunctionFragment &FF,
+ uint64_t DotAddress,
+ bool RecordAddresses) {
+ assert(shouldEmitFunctionFragment(BC, Func) &&
+ "attempting to lay out a function BinaryEmitter will not emit");
+
+ const FragmentNum Fragment = FF.getFragmentNum();
+ assert((Fragment == FragmentNum::main() || Fragment == FragmentNum::cold()) &&
+ "LongJmp supports only main and cold function fragments");
+ const bool IsCold = Fragment == FragmentNum::cold();
+
+ const bool HasFixedOutputAddress =
+ Func.isInjected() && Func.getOutputAddress();
+ const bool NeedsRelocationAlignment =
+ BC.HasRelocations && !HasFixedOutputAddress;
+ const bool NeedsNonRelocInjectedAlignment =
+ !BC.HasRelocations && Func.isInjected() && !HasFixedOutputAddress;
+ const bool NeedsNonRelocColdAlignment =
+ !BC.HasRelocations && !Func.isInjected() && IsCold;
+
+ // Apply the alignment that affects the fragment's mapped address.
+ // Section-relative fragments mirror BinaryEmitter::emitFunction(); moved
+ // cold fragments mirror mapCodeSectionsInPlace(). Ordinary non-relocation
+ // main fragments and fixed-address injected functions already have exact
+ // addresses.
+ if (NeedsRelocationAlignment) {
+ DotAddress = alignTo(DotAddress, Func.getMinAlignment());
+ const uint16_t MaxAlignmentBytes =
+ IsCold ? Func.getMaxColdAlignmentBytes() : Func.getMaxAlignmentBytes();
+ if (MaxAlignmentBytes > 0)
+ DotAddress =
+ applyCodeAlignment(DotAddress, Func.getAlign(), MaxAlignmentBytes);
+ } else if (NeedsNonRelocInjectedAlignment) {
+ // Newly allocated injected sections retain BinaryEmitter's regular
+ // non-relocation function alignment.
+ DotAddress = alignTo(DotAddress, Func.getAlign());
+ } else if (NeedsNonRelocColdAlignment) {
+ // mapCodeSectionsInPlace() aligns each moved cold fragment to a hard-coded
+ // 16-byte boundary.
+ DotAddress = alignTo(DotAddress, ColdFragmentAlignment);
+ }
+
+ // BinaryEmitter::emitFunction() places --pad-funcs-before after function
+ // alignment and rejects nonzero padding in non-relocation mode.
+ if (BC.HasRelocations)
+ DotAddress += opts::padFunctionBefore(Func);
+
+ // BinaryEmitter::emitFunction() emits the fragment entry symbols here.
+ if (RecordAddresses) {
+ if (!IsCold)
+ HotAddresses[&Func] = DotAddress;
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp layout: "
+ << (IsCold ? "cold" : "main") << " fragment "
+ << Func.getPrintName() << " starts at 0x"
+ << Twine::utohexstr(DotAddress) << '\n');
}
- // Ensure that tentative code layout always runs for cold blocks.
- if (!ColdLayoutDone)
- runColdLayout();
+ // --break-funcs emits UD2 before the function body.
+ DotAddress += opts::breakFunctionSize(Func);
+
+ DotAddress = layoutFunctionBody(BC, Func, FF, DotAddress, RecordAddresses);
+
+ // BinaryEmitter::emitFunction() emits --pad-funcs after the body in both
+ // relocation and non-relocation modes.
+ DotAddress += opts::padFunctionAfter(Func);
+
+ // --mark-funcs emits the target-specific trap marker after the fragment.
+ DotAddress += opts::markFunctionBytes(BC).size();
+
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: " << (IsCold ? "cold" : "main")
+ << " fragment " << Func.getPrintName() << " ends at 0x"
+ << Twine::utohexstr(DotAddress) << '\n';
+ });
+
+ return DotAddress;
+}
+
+uint64_t LongJmpPass::layoutSection(const BinaryContext &BC,
+ const SectionPlacement &Section,
+ uint64_t DotAddress, bool RecordAddresses) {
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: section " << Section.Name
+ << " starts at 0x" << Twine::utohexstr(DotAddress)
+ << ", alignment 0x" << Twine::utohexstr(Section.Alignment) << ", "
+ << Section.Fragments.size() << " fragments\n";
+ });
+
+ for (const FunctionFragmentPlacement &Placement : Section.Fragments) {
+ const BinaryFunction &Func = *Placement.Func;
- // BBs
- for (BinaryFunction *Func : SortedFunctions)
- tentativeBBLayout(*Func);
+ const FunctionFragment &FF =
+ Func.getLayout().getFragment(Placement.Fragment);
+ DotAddress =
+ layoutFunctionFragment(BC, Func, FF, DotAddress, RecordAddresses);
+ }
+
+ LLVM_DEBUG({
+ if (RecordAddresses)
+ dbgs() << "BOLT-DEBUG: LongJmp layout: section " << Section.Name
+ << " ends at 0x" << Twine::utohexstr(DotAddress) << '\n';
+ });
return DotAddress;
}
-void LongJmpPass::tentativeLayout(const BinaryContext &BC,
- BinaryFunctionListType &SortedFunctions) {
- uint64_t DotAddress = BC.LayoutStartAddress;
-
- if (!BC.HasRelocations) {
- for (BinaryFunction *Func : SortedFunctions) {
- HotAddresses[Func] = Func->getAddress();
- DotAddress = alignTo(DotAddress, ColdFragAlign);
- ColdAddresses[Func] = DotAddress;
- if (Func->isSplit())
- DotAddress += Func->estimateColdSize();
- tentativeBBLayout(*Func);
+uint64_t LongJmpPass::layoutSectionsForward(const BinaryContext &BC,
+ uint64_t DotAddress) {
+ const bool AdjustMainSection =
+ BC.HasRelocations &&
+ (opts::HotText || (opts::Hugify && !BC.HasFixedLoadAddress));
+ std::optional<uint64_t> MainSectionEnd = std::nullopt;
+
+ // Mirror allocateAt() in RewriteInstance::mapCodeSections().
+ for (const SectionPlacement &Section : Sections) {
+ DotAddress = alignTo(DotAddress, Section.Alignment);
+ DotAddress = layoutSection(BC, Section, DotAddress);
+
+ if (AdjustMainSection &&
+ StringRef(Section.Name) == BC.getMainCodeSectionName()) {
+ if (opts::HotText)
+ MainSectionEnd = DotAddress;
+ // LongJmp supports only main and cold fragments. Mirror the extra
+ // post-main alignment in allocateAt() for --hugify.
+ if (opts::Hugify && !BC.HasFixedLoadAddress)
+ DotAddress = alignTo(DotAddress, Section.Alignment);
}
+ }
- return;
+ // Mirror RewriteInstance::mapCodeSections() padding used to accommodate
+ // hot-text huge-page mapping. LongJmp does not support warm fragments, so the
+ // hot-text end is the end of the main section. RewriteInstance applies this
+ // adjustment only in allocateAt() to advance the next free address;
+ // allocateBefore() starts from a fixed upper boundary and has no
+ // corresponding adjustment.
+ if (MainSectionEnd)
+ DotAddress = std::max(DotAddress, alignTo(*MainSectionEnd, BC.PageAlign));
+
+ return DotAddress;
+}
+
+bool LongJmpPass::layoutSectionsBackward(const BinaryContext &BC,
+ uint64_t DotAddress) {
+ SmallVector<uint64_t, 4> SectionAddresses(Sections.size());
+ // Mirror allocateBefore() in RewriteInstance::mapCodeSections(): assign
+ // section bases in reverse while preserving their sorted output order.
+ for (size_t I = Sections.size(); I > 0; --I) {
+ const SectionPlacement &Section = Sections[I - 1];
+ uint64_t &SectionAddress = SectionAddresses[I - 1];
+ // Match the BinarySection::getOutputSize() consumed by allocateBefore().
+ const uint64_t SectionSize =
+ layoutSection(BC, Section, 0, /*RecordAddresses=*/false);
+ if (SectionSize > DotAddress)
+ return false;
+ DotAddress -= SectionSize;
+ DotAddress = alignDown(DotAddress, Section.Alignment);
+ if (DotAddress < BC.OldTextSectionAddress)
+ return false;
+ SectionAddress = DotAddress;
}
- // Relocation mode
- uint64_t EstimatedTextSize = 0;
- if (opts::UseOldText) {
- EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0);
-
- // Initial padding
- if (EstimatedTextSize <= BC.OldTextSectionSize) {
- DotAddress = BC.OldTextSectionAddress;
- uint64_t Pad = offsetToAlignment(DotAddress, llvm::Align(BC.AlignText));
- if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) {
- DotAddress += Pad;
+ // Contents within every section are still laid out toward higher addresses.
+ for (size_t I = 0; I < Sections.size(); ++I)
+ layoutSection(BC, Sections[I], SectionAddresses[I]);
+
+ return true;
+}
+
+void LongJmpPass::layoutFunctions(
+ const BinaryContext &BC, const BinaryFunctionListType &SortedFunctions) {
+ if (BC.HasRelocations) {
+ // Mirror the old-text allocation choice in
+ // RewriteInstance::mapCodeSections().
+ bool AllocatedAtOldText = false;
+ if (opts::UseOldText) {
+ if (opts::HotFunctionsAtEnd) {
+ AllocatedAtOldText = layoutSectionsBackward(
+ BC, BC.OldTextSectionAddress + BC.OldTextSectionSize);
+ } else {
+ const uint64_t EndAddress =
+ layoutSectionsForward(BC, BC.OldTextSectionAddress);
+ AllocatedAtOldText =
+ EndAddress <= BC.OldTextSectionAddress + BC.OldTextSectionSize;
}
+
+ if (!AllocatedAtOldText) {
+ BC.errs() << "BOLT-WARNING: --use-old-text failed during LongJmp "
+ "layout. The original .text is too small to fit the new "
+ "code.\n";
+ // Do not clear opts::UseOldText here. RewriteInstance also uses it
+ // during emission to decide how to handle non-code sections such as
+ // .eh_frame, and performs the authoritative fallback later.
+ } else {
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: The layout fits into the "
+ "original .text section\n");
+ }
+ }
+
+ // mapCodeSections() falls back to allocateAt() when old text is unused
+ // or too small.
+ if (!AllocatedAtOldText)
+ layoutSectionsForward(BC, BC.LayoutStartAddress);
+ } else {
+ // Mirror RewriteInstance::mapCodeSectionsInPlace(). Main fragments retain
+ // their input addresses, while split cold fragments are appended in
+ // original-function order starting at the first free output address.
+ uint64_t ColdAddress = BC.LayoutStartAddress;
+ for (const auto &BFI : BC.getBinaryFunctions()) {
+ const BinaryFunction &Func = BFI.second;
+
+ // PopulateOutputFunctions excludes functions for which shouldEmit()
+ // returns false. LongJmp never relaxes them, so they need no entries in
+ // the function or basic-block address maps. Unlike the other layout
+ // loops, this one visits getBinaryFunctions() to mirror
+ // mapCodeSectionsInPlace(), and therefore needs an explicit check.
+ if (!BC.shouldEmit(Func))
+ continue;
+
+ if (!shouldEmitFunctionFragment(BC, Func))
+ continue;
+
+ layoutFunctionFragment(BC, Func, Func.getLayout().getMainFragment(),
+ Func.getAddress());
+
+ if (Func.isSplit()) {
+ assert(Func.getLayout().isHotColdSplit() &&
+ "non-relocation mode supports only hot/cold splitting");
+ ColdAddress = layoutFunctionFragment(
+ BC, Func, Func.getLayout().getFragment(FragmentNum::cold()),
+ ColdAddress);
+ }
+ }
+
+ // mapCodeSectionsInPlace() allocates non-fixed injected sections, normally
+ // .text.injected, immediately after the moved cold fragments. These are
+ // the only entries in Sections in non-relocation mode.
+ layoutSectionsForward(BC, ColdAddress);
+ }
+
+ // Fixed-address injected functions are outside Sections and use their
+ // pre-assigned output addresses.
+ for (const BinaryFunction *Func : SortedFunctions) {
+ if (!shouldEmitFunctionFragment(BC, *Func))
+ continue;
+
+ if (Func->isInjected() && Func->getOutputAddress()) {
+ assert(!Func->isSplit() && "injected functions cannot be split");
+ layoutFunctionFragment(BC, *Func, Func->getLayout().getMainFragment(),
+ Func->getOutputAddress());
}
}
+}
- if (!EstimatedTextSize || EstimatedTextSize > BC.OldTextSectionSize) {
- uint64_t TextAlign =
- std::max<uint64_t>(BC.AlignText, BC.MaxMainCodeAlignment.load());
- DotAddress = alignTo(BC.LayoutStartAddress, TextAlign);
+void LongJmpPass::layout(const BinaryContext &BC,
+ const BinaryFunctionListType &SortedFunctions) {
+ HotAddresses.clear();
+ BBAddresses.clear();
+ Sections.clear();
+
+ LLVM_DEBUG(
+ dbgs() << "BOLT-DEBUG: LongJmp layout starts at 0x"
+ << Twine::utohexstr(BC.LayoutStartAddress) << ", text alignment 0x"
+ << Twine::utohexstr(BC.AlignText) << ", function alignment 0x"
+ << Twine::utohexstr(BC.AlignFunctions)
+ << ", maximum main alignment 0x"
+ << Twine::utohexstr(BC.MaxMainCodeAlignment.load())
+ << ", maximum cold alignment 0x"
+ << Twine::utohexstr(BC.MaxColdCodeAlignment.load()) << '\n');
+
+ // Reproduce the code placement performed later by BinaryEmitter and
+ // RewriteInstance. First catalogue fragments whose addresses are determined
+ // by output-section placement. In relocation mode this includes all emitted
+ // fragments except fixed-address injected patches. In non-relocation mode it
+ // includes only non-fixed injected functions; ordinary main fragments remain
+ // at their input addresses, while mapCodeSectionsInPlace() allocates moved
+ // cold fragments directly.
+ //
+ // Section alignment depends on every fragment assigned to the section, so
+ // the complete catalogue must be built before calculating any section base.
+ // The layout phase then mirrors either mapCodeSections() or
+ // mapCodeSectionsInPlace(), and finally records fixed injected patches whose
+ // addresses do not come from the section catalogue.
+
+ assignFunctionsToSections(BC, SortedFunctions);
+
+ if (BC.HasRelocations) {
+ const CodeSectionOrder CompareSections(BC);
+
+ // Mirror RewriteInstance::getCodeSections(). Sections not named explicitly
+ // retain their first-emission order.
+ llvm::stable_sort(
+ Sections, [&](const SectionPlacement &A, const SectionPlacement &B) {
+ return CompareSections(A.Name, B.Name);
+ });
}
- tentativeLayoutRelocMode(BC, SortedFunctions, DotAddress);
+ layoutFunctions(BC, SortedFunctions);
}
bool LongJmpPass::usesStub(const BinaryFunction &Func,
@@ -520,7 +836,7 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
const MCSymbol *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB.begin());
const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(RealTargetSym);
uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB);
- uint64_t DotAddress = BBAddresses[&StubBB];
+ uint64_t DotAddress = BBAddresses.at(&StubBB);
uint64_t PCRelTgtAddress = DotAddress > TgtAddress ? DotAddress - TgtAddress
: TgtAddress - DotAddress;
@@ -533,10 +849,9 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
if (Bits >= RangeShortJmp)
return Error::success();
- LLVM_DEBUG(dbgs() << "Relaxing stub to short jump. PCRelTgtAddress = "
- << Twine::utohexstr(PCRelTgtAddress)
- << " RealTargetSym = " << RealTargetSym->getName()
- << "\n");
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: relaxing stub to short jump; "
+ << "distance 0x" << Twine::utohexstr(PCRelTgtAddress)
+ << ", target " << RealTargetSym->getName() << '\n');
relaxStubToShortJmp(StubBB, RealTargetSym);
StubBits[&StubBB] = RangeShortJmp;
Modified = true;
@@ -549,9 +864,9 @@ Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {
return createFatalBOLTError(
"BOLT-ERROR: Unable to relax stub for PIC binary\n");
- LLVM_DEBUG(dbgs() << "Relaxing stub to long jump. PCRelTgtAddress = "
- << Twine::utohexstr(PCRelTgtAddress)
- << " RealTargetSym = " << RealTargetSym->getName() << "\n");
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: relaxing stub to long jump; "
+ << "distance 0x" << Twine::utohexstr(PCRelTgtAddress)
+ << ", target " << RealTargetSym->getName() << '\n');
relaxStubToLongJmp(StubBB, RealTargetSym);
StubBits[&StubBB] = static_cast<int>(BC.AsmInfo->getCodePointerSize() * 8);
Modified = true;
@@ -582,26 +897,41 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,
uint64_t PCRelTgtAddress = getSymbolAddress(BC, TgtSym, TgtBB);
int64_t PCOffset = (int64_t)(PCRelTgtAddress - DotAddress);
- return PCOffset < MinVal || PCOffset > MaxVal;
+ const bool Result = PCOffset < MinVal || PCOffset > MaxVal;
+ LLVM_DEBUG({
+ if (Result)
+ dbgs() << "BOLT-DEBUG: LongJmp: out-of-range branch in "
+ << Func.getPrintName() << ", basic block " << BB.getName()
+ << ", source 0x" << Twine::utohexstr(DotAddress) << ", target "
+ << TgtSym->getName() << " at 0x"
+ << Twine::utohexstr(PCRelTgtAddress) << ", displacement "
+ << PCOffset << ", range [" << MinVal << ", " << MaxVal << "]\n";
+ });
+ return Result;
}
Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) {
const BinaryContext &BC = Func.getBinaryContext();
assert(BC.isAArch64() && "Unsupported arch");
+ // Keep the relaxation traversal consistent with layout(): functions that
+ // BinaryEmitter will not emit have no entries in BBAddresses.
+ if (!shouldEmitFunctionFragment(BC, Func))
+ return Error::success();
+
constexpr int InsnSize = 4; // AArch64
std::vector<std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>>>
Insertions;
BinaryBasicBlock *Frontier = getBBAtHotColdSplitPoint(Func);
- uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;
+ uint64_t FrontierAddress = Frontier ? BBAddresses.at(Frontier) : 0;
if (FrontierAddress)
FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;
// Add necessary stubs for branch targets we know we can't fit in the
// instruction
for (BinaryBasicBlock &BB : Func) {
- uint64_t DotAddress = BBAddresses[&BB];
+ uint64_t DotAddress = BBAddresses.at(&BB);
// Stubs themselves are relaxed on the next loop
if (Stubs[&Func].count(&BB))
continue;
@@ -1386,7 +1716,9 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
do {
++Iterations;
Modified = false;
- tentativeLayout(BC, Sorted);
+ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp: layout iteration " << Iterations
+ << '\n');
+ layout(BC, Sorted);
updateStubGroups();
for (BinaryFunction *Func : Sorted) {
if (auto E = relax(*Func, Modified))
diff --git a/bolt/lib/Passes/ReorderFunctions.cpp b/bolt/lib/Passes/ReorderFunctions.cpp
index f38f502e75ced..c2fb8d10614fc 100644
--- a/bolt/lib/Passes/ReorderFunctions.cpp
+++ b/bolt/lib/Passes/ReorderFunctions.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/ReorderFunctions.h"
+#include "bolt/Core/BinaryEmitter.h"
#include "bolt/Passes/HFSort.h"
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/STLExtras.h"
@@ -29,9 +30,6 @@ extern cl::OptionCategory BoltOptCategory;
extern cl::opt<unsigned> Verbosity;
extern cl::opt<uint32_t> RandomSeed;
-extern size_t padFunctionBefore(const bolt::BinaryFunction &Function);
-extern size_t padFunctionAfter(const bolt::BinaryFunction &Function);
-
extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions(
"reorder-functions",
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index cede4d2c946b5..45c7f5226680b 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4466,105 +4466,13 @@ void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
}
}
-namespace {
-
-/// Defines the strict weak ordering for BOLT-produced code sections.
-class CodeSectionOrder {
-public:
- CodeSectionOrder(StringRef ColdSectionName, StringRef HotTextMoverSectionName,
- StringRef MainSectionName, StringRef WarmSectionName,
- bool HotText, bool HotFunctionsAtEnd)
- : ColdSectionName(ColdSectionName),
- HotTextMoverSectionName(HotTextMoverSectionName),
- MainSectionName(MainSectionName), WarmSectionName(WarmSectionName),
- HotText(HotText), HotFunctionsAtEnd(HotFunctionsAtEnd) {}
-
- bool operator()(StringRef AName, StringRef BName) const {
- const SectionKind AKind = getKind(AName);
- const SectionKind BKind = getKind(BName);
- const unsigned ARank = getRank(AKind);
- const unsigned BRank = getRank(BKind);
- if (ARank != BRank)
- return ARank < BRank;
-
- if (AKind == SectionKind::Cold) {
- if (AName.size() != BName.size())
- return HotFunctionsAtEnd ? AName.size() > BName.size()
- : AName.size() < BName.size();
- if (AName != BName)
- return HotFunctionsAtEnd ? AName > BName : AName < BName;
- }
-
- return false;
- }
-
-private:
- enum class SectionKind { Mover, Main, Warm, Cold, Other };
-
- SectionKind getKind(StringRef Name) const {
- if (HotText && Name == HotTextMoverSectionName)
- return SectionKind::Mover;
- if (Name == MainSectionName)
- return SectionKind::Main;
- if (Name == WarmSectionName)
- return SectionKind::Warm;
- if (Name.starts_with(ColdSectionName))
- return SectionKind::Cold;
- return SectionKind::Other;
- }
-
- unsigned getRank(SectionKind Kind) const {
- if (Kind == SectionKind::Mover)
- return 0;
- if (HotFunctionsAtEnd) {
- switch (Kind) {
- case SectionKind::Other:
- return 1;
- case SectionKind::Cold:
- return 2;
- case SectionKind::Warm:
- return 3;
- case SectionKind::Main:
- return 4;
- case SectionKind::Mover:
- llvm_unreachable("handled above");
- }
- }
- switch (Kind) {
- case SectionKind::Main:
- return 1;
- case SectionKind::Warm:
- return 2;
- case SectionKind::Cold:
- return 3;
- case SectionKind::Other:
- return 4;
- case SectionKind::Mover:
- llvm_unreachable("handled above");
- }
- llvm_unreachable("unknown section kind");
- }
-
- StringRef ColdSectionName;
- StringRef HotTextMoverSectionName;
- StringRef MainSectionName;
- StringRef WarmSectionName;
- bool HotText;
- bool HotFunctionsAtEnd;
-};
-
-} // namespace
-
std::vector<BinarySection *> RewriteInstance::getCodeSections() {
std::vector<BinarySection *> CodeSections;
for (BinarySection &Section : BC->textSections())
if (Section.hasValidSectionID())
CodeSections.emplace_back(&Section);
- const CodeSectionOrder CompareSections(
- BC->getColdCodeSectionName(), BC->getHotTextMoverSectionName(),
- BC->getMainCodeSectionName(), BC->getWarmCodeSectionName(), opts::HotText,
- opts::HotFunctionsAtEnd);
+ const CodeSectionOrder CompareSections(*BC);
// Determine the order of sections.
llvm::stable_sort(CodeSections,
@@ -4781,21 +4689,36 @@ void RewriteInstance::mapCodeSectionsInPlace(
// Processing in non-relocation mode.
uint64_t NewTextSectionStartAddress = NextAvailableAddress;
+ auto MapMainFragment = [&](BinaryFunction &Func, uint64_t OutputAddress) {
+ ErrorOr<BinarySection &> FuncSection = Func.getCodeSection();
+ assert(FuncSection && "cannot find section for function");
+ FuncSection->setOutputAddress(OutputAddress);
+ LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
+ << Twine::utohexstr(FuncSection->getAllocAddress())
+ << " to 0x" << Twine::utohexstr(OutputAddress) << '\n');
+ MapSection(*FuncSection, OutputAddress);
+ Func.setImageAddress(FuncSection->getAllocAddress());
+ Func.setImageSize(FuncSection->getOutputSize());
+ };
+
+ // Map injected patches at their pre-assigned addresses. Injected functions
+ // returned by BC->getInjectedBinaryFunctions() are not contained in
+ // BC->getBinaryFunctions(). Their sections are unique and are removed after
+ // their contents have been copied in place.
+ for (BinaryFunction *Func : BC->getInjectedBinaryFunctions()) {
+ const uint64_t OutputAddress = Func->getOutputAddress();
+ if (!Func->isEmitted() || !OutputAddress)
+ continue;
+
+ MapMainFragment(*Func, OutputAddress);
+ }
+
for (auto &BFI : BC->getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
if (!Function.isEmitted())
continue;
- ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();
- assert(FuncSection && "cannot find section for function");
- FuncSection->setOutputAddress(Function.getAddress());
- LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
- << Twine::utohexstr(FuncSection->getAllocAddress())
- << " to 0x" << Twine::utohexstr(Function.getAddress())
- << '\n');
- MapSection(*FuncSection, Function.getAddress());
- Function.setImageAddress(FuncSection->getAllocAddress());
- Function.setImageSize(FuncSection->getOutputSize());
+ MapMainFragment(Function, Function.getAddress());
assert(Function.getImageSize() <= Function.getMaxSize() &&
"Unexpected large function");
@@ -4848,6 +4771,37 @@ void RewriteInstance::mapCodeSectionsInPlace(
Section.setOutputFileOffset(
getFileOffsetForAddress(NewTextSectionStartAddress));
}
+
+ // Unlike cold fragments, non-fixed injected functions remain in their
+ // emitted code sections. Allocate those sections immediately after the
+ // moved cold fragments so their placement depends only on code layout. The
+ // functions themselves receive their final output addresses later from
+ // linker-resolved symbols in BinaryFunction::updateOutputValues().
+ SmallVector<BinarySection *, 4> InjectedSections;
+ for (const BinaryFunction *Func : BC->getOutputBinaryFunctions()) {
+ if (!(Func->isInjected() && Func->isEmitted() && !Func->getOutputAddress()))
+ continue;
+
+ assert(!Func->isSplit() && "injected functions cannot be split");
+ ErrorOr<BinarySection &> Section = Func->getCodeSection();
+ if (Section && !llvm::is_contained(InjectedSections, &*Section))
+ InjectedSections.push_back(&*Section);
+ }
+
+ for (BinarySection *Section : InjectedSections) {
+ assert(!Section->getOutputAddress() &&
+ "non-fixed injected section already mapped");
+ NextAvailableAddress =
+ alignTo(NextAvailableAddress, Section->getAlignment());
+ LLVM_DEBUG(
+ dbgs() << "BOLT: mapping injected section " << Section->getName()
+ << " at 0x" << Twine::utohexstr(Section->getAllocAddress())
+ << " to 0x" << Twine::utohexstr(NextAvailableAddress) << "\n");
+ MapSection(*Section, NextAvailableAddress);
+ Section->setOutputAddress(NextAvailableAddress);
+ Section->setOutputFileOffset(getFileOffsetForAddress(NextAvailableAddress));
+ NextAvailableAddress += Section->getOutputSize();
+ }
}
void RewriteInstance::mapAllocatableSections(
diff --git a/bolt/test/AArch64/LongJmp-Layout/basic.s b/bolt/test/AArch64/LongJmp-Layout/basic.s
new file mode 100644
index 0000000000000..1eeeb59841c51
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/basic.s
@@ -0,0 +1,106 @@
+## Check that LongJmp's layout matches actual emission for ordinary
+## functions, function and basic-block alignment, explicit padding, and the
+## optional function boundary markers.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.default.bolt --lite=0 \
+# RUN: --debug-only=longjmp > %t.default.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.default.bolt >> %t.default.log
+# RUN: FileCheck %s --check-prefix=DEFAULT < %t.default.log
+# RUN: llvm-bolt %t.exe -o %t.align.bolt --lite=0 \
+# RUN: --align-text=4096 --align-functions=256 \
+# RUN: --align-functions-max-bytes=255 --preserve-blocks-alignment \
+# RUN: --debug-only=longjmp > %t.align.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.align.bolt >> %t.align.log
+# RUN: FileCheck %s --check-prefix=ALIGN < %t.align.log
+# RUN: llvm-bolt %t.exe -o %t.max-align.bolt --lite=0 \
+# RUN: --align-functions=256 --align-functions-max-bytes=1 \
+# RUN: --debug-only=longjmp > %t.max-align.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.max-align.bolt >> %t.max-align.log
+# RUN: FileCheck %s --check-prefix=MAX-ALIGN < %t.max-align.log
+# RUN: llvm-bolt %t.exe -o %t.padding.bolt --lite=0 \
+# RUN: --pad-funcs-before=second:20 --pad-funcs=first:12 \
+# RUN: --break-funcs=first,second --mark-funcs \
+# RUN: --debug-only=longjmp > %t.padding.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.padding.bolt >> %t.padding.log
+# RUN: FileCheck %s --check-prefix=PADDING < %t.padding.log
+# RUN: %clang %cflags %t.o -o %t.noreloc.exe -nostdlib
+# RUN: llvm-bolt %t.noreloc.exe -o %t.noreloc.bolt --lite=0 \
+# RUN: --preserve-blocks-alignment --debug-only=longjmp \
+# RUN: > %t.noreloc.log 2>&1
+# RUN: FileCheck %s --check-prefix=NONRELOC < %t.noreloc.log
+
+# DEFAULT: LongJmp layout: section .text starts at 0x[[TEXT:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# DEFAULT: LongJmp layout: section .text ends at 0x
+# DEFAULT: _start T [[START]]
+# DEFAULT: first T [[FIRST]]
+# DEFAULT: second T [[SECOND]]
+
+# ALIGN: LongJmp layout: section .text starts at 0x[[TEXT:[0-9a-f]+]], alignment 0x1000
+# ALIGN: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# ALIGN: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
+# ALIGN: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# ALIGN: _start T [[START]]
+# ALIGN: first T [[FIRST]]
+# ALIGN: second T [[SECOND]]
+
+# MAX-ALIGN: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# MAX-ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# MAX-ALIGN: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# MAX-ALIGN: _start T [[START]]
+# MAX-ALIGN: first T [[FIRST]]
+# MAX-ALIGN: second T [[SECOND]]
+
+# PADDING: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# PADDING: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# PADDING: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
+# PADDING: _start T [[START]]
+# PADDING: first T [[FIRST]]
+# PADDING: second T [[SECOND]]
+
+# NONRELOC: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# NONRELOC: BOLT-DEBUG: LongJmp layout starts at 0x
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x[[FIRST]]
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]*[02468ace]0}}
+
+ .text
+ .p2align 6
+ .globl _start
+ .type _start, %function
+_start:
+ bl first
+ bl second
+ ret
+ .size _start, .-_start
+
+ .p2align 6
+ .globl first
+ .type first, %function
+first:
+ cbz x0, .Lfirst_aligned
+ add x0, x0, #1
+ ret
+ .p2align 5
+.Lfirst_aligned:
+ sub x0, x0, #1
+ ret
+ .size first, .-first
+
+ .p2align 7
+ .globl second
+ .type second, %function
+second:
+ add x1, x1, #1
+ ret
+ .size second, .-second
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/constant-island.s b/bolt/test/AArch64/LongJmp-Layout/constant-island.s
new file mode 100644
index 0000000000000..b007849fa1b25
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/constant-island.s
@@ -0,0 +1,61 @@
+## Check layout of an aligned constant island. With splitting, the
+## cold block's reference also exercises duplicated constant-island emission.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.unsplit.bolt --data %t.fdata --lite=0 \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.unsplit.bolt | FileCheck %s \
+# RUN: --check-prefix=UNSPLIT-SECTIONS \
+# RUN: --implicit-check-not='.text.cold'
+# RUN: llvm-bolt %t.exe -o %t.split.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold --debug-only=longjmp 2>&1 \
+# RUN: | FileCheck %s
+# RUN: llvm-readelf -S %t.split.bolt | FileCheck %s \
+# RUN: --check-prefix=SPLIT-SECTIONS
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# UNSPLIT-SECTIONS: .text
+# SPLIT-SECTIONS: .text
+# SPLIT-SECTIONS: .text.cold
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl island_user
+ ret
+ .size _start, .-_start
+
+ .globl island_user
+ .type island_user, %function
+island_user:
+.entry_island_user:
+# FDATA: 1 island_user #.entry_island_user# 10
+ cbz x0, .Lcold
+.hot_island_user:
+# FDATA: 1 island_user #.hot_island_user# 10
+ ret
+.Lcold:
+ adr x1, .Lconstant_island
+ ldr x1, [x1]
+ ret
+ .size island_user, .-island_user
+
+ .p2align 6
+.Lconstant_island:
+ .xword 0x1122334455667788
+ .xword 0x8877665544332211
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/empty-function.s b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
new file mode 100644
index 0000000000000..8591a8410bf1d
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
@@ -0,0 +1,99 @@
+## Check the two kinds of empty output handled by BinaryEmitter and mirrored by
+## LongJmp. A zero-sized input function is not emitted at all. A function that
+## becomes instruction-less is still emitted in relocation mode to define its
+## main symbol, but is omitted when overwriting the original text in place.
+## Structurally empty split fragments never produce symbols.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+# RUN: %t/empty-function.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 \
+# RUN: --pad-funcs-before=empty:64 --pad-funcs=empty:64 \
+# RUN: --debug-only=longjmp > %t.log 2>&1
+# RUN: FileCheck %s --check-prefix=RELOC \
+# RUN: --implicit-check-not="LongJmp layout: main fragment empty" < %t.log
+# RUN: llvm-nm %t.bolt | FileCheck %s --check-prefix=MAIN-SYMBOL
+# RUN: %clang %cflags %t.o -o %t.noreloc.exe -nostdlib
+# RUN: llvm-bolt %t.noreloc.exe -o %t.noreloc.bolt --lite=0 \
+# RUN: --pad-funcs-before=empty:64 --pad-funcs=empty:64 \
+# RUN: --debug-only=longjmp > %t.noreloc.log 2>&1
+# RUN: FileCheck %s --check-prefix=NONRELOC \
+# RUN: --implicit-check-not="LongJmp layout: main fragment empty" \
+# RUN: --implicit-check-not="LongJmp layout: main fragment nop_only" \
+# RUN: < %t.noreloc.log
+
+## --split-strategy=all deliberately leaves gaps in the fragment numbering
+## when the secondary entry is forced back into the main fragment. This creates
+## structurally empty fragments 1 and 2 followed by non-empty fragment 3.
+## Compact code model bypasses the main/cold-only LongJmp layout path, allowing
+## this subcase to check BinaryEmitter empty-fragment handling directly.
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+# RUN: %t/empty-fragments.s -o %t.fragments.o
+# RUN: %clang %cflags %t.fragments.o -o %t.fragments.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.fragments.exe -o %t.fragments.bolt --lite=0 \
+# RUN: --split-functions --split-strategy=all --compact-code-model
+# RUN: llvm-nm %t.fragments.bolt | FileCheck %s --check-prefix=FRAGMENTS \
+# RUN: --implicit-check-not="_start.cold.0" \
+# RUN: --implicit-check-not="_start.cold.1"
+
+# RELOC: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}, alignment 0x{{[0-9a-f]+}}, 3 fragments
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment next starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only starts at 0x
+# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only ends at 0x
+
+# MAIN-SYMBOL: T nop_only
+
+# NONRELOC: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment next starts at 0x
+
+# FRAGMENTS: t _start.cold.2
+
+#--- empty-function.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl next
+ ret
+ .size _start, .-_start
+
+ .globl next
+ .type next, %function
+next:
+ ret
+ .size next, .-next
+
+ .globl nop_only
+ .type nop_only, %function
+nop_only:
+ nop
+ .size nop_only, .-nop_only
+
+ .globl empty
+ .type empty, %function
+empty:
+ .size empty, .-empty
+
+ .reloc 0, R_AARCH64_NONE
+
+#--- empty-fragments.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ cbz x0, secondary_entry
+ ret
+ .globl secondary_entry
+secondary_entry:
+ add x0, x0, #1
+ b .Llast
+.Llast:
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s b/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
new file mode 100644
index 0000000000000..b8cd051e0673a
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/fixed-injected.s
@@ -0,0 +1,31 @@
+## Check that LongJmp uses the pre-assigned address of an injected patch.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --use-old-text=0 \
+# RUN: --force-patch --debug-only=longjmp > %t.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment patched.org.0/ starts at 0x[[PATCH:[0-9a-f]+]]
+# CHECK: patched.org.0 t [[PATCH]]
+
+ .text
+ .balign 4
+ .globl patched
+ .type patched, %function
+patched:
+ .rept 32
+ nop
+ .endr
+ ret
+ .size patched, .-patched
+
+ .globl _start
+ .type _start, %function
+_start:
+ bl patched
+ ret
+ .size _start, .-_start
diff --git a/bolt/test/AArch64/LongJmp-Layout/function-padding.s b/bolt/test/AArch64/LongJmp-Layout/function-padding.s
new file mode 100644
index 0000000000000..1445eb8ce9de6
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/function-padding.s
@@ -0,0 +1,55 @@
+## Check that LongJmp includes padding before and after functions in its
+## layout. Either padding request puts target outside the range of
+## foo's direct call and requires a stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.after.bolt --data %t.fdata --lite=0 \
+# RUN: --pad-funcs=foo:134217728 2>&1 | FileCheck %s --check-prefix=MAIN
+# RUN: llvm-bolt %t.exe -o %t.before.bolt --data %t.fdata --lite=0 \
+# RUN: --pad-funcs-before=target:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=MAIN
+# RUN: llvm-bolt %t.exe -o %t.split-before.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold \
+# RUN: --pad-funcs-before=target:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SPLIT-BEFORE
+# RUN: llvm-bolt %t.exe -o %t.split-after.bolt --data %t.fdata --lite=0 \
+# RUN: --split-functions --split-all-cold --pad-funcs=foo:134217728 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SPLIT-AFTER
+
+# MAIN: BOLT-INFO: Inserted 1 stubs in the hot area and 0 stubs in the cold area.
+# SPLIT-BEFORE: BOLT-INFO: Inserted 0 stubs in the hot area and 1 stubs in the cold area.
+# SPLIT-AFTER: BOLT-INFO: Inserted 1 stubs in the hot area and 1 stubs in the cold area.
+
+ .text
+ .globl foo
+ .type foo, %function
+foo:
+.entry_foo:
+# FDATA: 1 foo #.entry_foo# 10
+ cbnz x0, .hot_foo
+.Lcold:
+ bl target
+ ret
+.hot_foo:
+# FDATA: 1 foo #.hot_foo# 10
+ ret
+ .size foo, .-foo
+
+ .globl target
+ .type target, %function
+target:
+ ret
+ .size target, .-target
+
+ .globl _start
+ .type _start, %function
+_start:
+ nop
+ ret
+
+ .size _start, .-_start
diff --git a/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s b/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
new file mode 100644
index 0000000000000..79b862d4cc26f
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/hot-functions-at-end.s
@@ -0,0 +1,105 @@
+## Check that LongJmp mirrors code-section placement when cold sections are
+## placed before hot sections with --hot-functions-at-end, including the
+## backward section allocation used by --use-old-text.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.hot.bolt --data %t.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --hot-functions-at-end \
+# RUN: --align-text=16 --align-functions=4 --debug-only=longjmp \
+# RUN: > %t.hot.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.hot.bolt >> %t.hot.log
+# RUN: FileCheck %s --check-prefix=HOT < %t.hot.log
+# RUN: llvm-bolt %t.exe -o %t.old.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --reorder-functions=exec-count \
+# RUN: --hot-functions-at-end --align-text=4 --align-functions=4 \
+# RUN: --debug-only=longjmp > %t.old.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.old.bolt >> %t.old.log
+# RUN: FileCheck %s --check-prefix=OLD < %t.old.log
+# RUN: llvm-bolt %t.exe -o %t.old-forward.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --hot-text=0 --reorder-functions=exec-count \
+# RUN: --align-text=4 --align-functions=4 --debug-only=longjmp \
+# RUN: > %t.old-forward.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.old-forward.bolt >> %t.old-forward.log
+# RUN: FileCheck %s --check-prefix=OLD-FORWARD < %t.old-forward.log
+# RUN: llvm-bolt %t.exe -o %t.old-forward-fail.bolt --data %t.fdata \
+# RUN: --lite=0 --use-old-text --hot-text=0 \
+# RUN: --reorder-functions=exec-count --align-text=4 --align-functions=4 \
+# RUN: --pad-funcs=_start:1024 --debug-only=longjmp 2>&1 | \
+# RUN: FileCheck %s --check-prefix=OLD-FORWARD-FAIL
+# RUN: llvm-bolt %t.exe -o %t.old-fail.bolt --data %t.fdata --lite=0 \
+# RUN: --use-old-text --reorder-functions=exec-count \
+# RUN: --hot-functions-at-end --align-text=4 --align-functions=4 \
+# RUN: --pad-funcs=_start:1024 --debug-only=longjmp 2>&1 | \
+# RUN: FileCheck %s --check-prefix=OLD-FAIL
+
+# HOT: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x{{[0-9a-f]+}}
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[COLD:[0-9a-f]+]]
+# HOT: BOLT-DEBUG: LongJmp layout: section .text.cold ends at 0x
+# HOT: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# HOT: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[HOTFUNC:[0-9a-f]+]]
+# HOT: cold_function T [[COLD]]
+# HOT: _start T [[START]]
+# HOT: hot T [[HOTFUNC]]
+
+# OLD: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x{{[0-9a-f]+}}
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[OLD_COLD:[0-9a-f]+]]
+# OLD: BOLT-DEBUG: LongJmp layout: section .text.cold ends at 0x
+# OLD: BOLT-DEBUG: LongJmp layout: section .text starts at 0x{{[0-9a-f]+}}
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[OLD_START:[0-9a-f]+]]
+# OLD: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[OLD_HOT:[0-9a-f]+]]
+# OLD: BOLT-INFO: using original .text for new code
+# OLD: cold_function T [[OLD_COLD]]
+# OLD: _start T [[OLD_START]]
+# OLD: hot T [[OLD_HOT]]
+
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[FORWARD_START:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment hot starts at 0x[[FORWARD_HOT:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: section .text.cold starts at 0x
+# OLD-FORWARD: BOLT-DEBUG: LongJmp layout: main fragment cold_function starts at 0x[[FORWARD_COLD:[0-9a-f]+]]
+# OLD-FORWARD: BOLT-INFO: using original .text for new code
+# OLD-FORWARD: _start T [[FORWARD_START]]
+# OLD-FORWARD: hot T [[FORWARD_HOT]]
+# OLD-FORWARD: cold_function T [[FORWARD_COLD]]
+
+# OLD-FORWARD-FAIL: BOLT-WARNING: --use-old-text failed during LongJmp layout.
+# OLD-FORWARD-FAIL: BOLT-WARNING: --use-old-text failed. The original .text
+# OLD-FORWARD-FAIL-NOT: BOLT-INFO: using original .text for new code
+
+# OLD-FAIL: BOLT-WARNING: --use-old-text failed during LongJmp layout.
+# OLD-FAIL: BOLT-WARNING: --use-old-text failed. The original .text
+# OLD-FAIL-NOT: BOLT-INFO: using original .text for new code
+
+ .text
+ .space 512, 0
+
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+ bl hot
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+.entry_hot:
+ ret
+ .size hot, .-hot
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
+
+# FDATA: 1 _start #.entry_start# 10
+# FDATA: 1 hot #.entry_hot# 10
diff --git a/bolt/test/AArch64/LongJmp-Layout/hugify.s b/bolt/test/AArch64/LongJmp-Layout/hugify.s
new file mode 100644
index 0000000000000..ab9fa02822864
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/hugify.s
@@ -0,0 +1,48 @@
+## Check the extra alignment between the main and cold code sections inserted
+## for --hugify.
+
+# REQUIRES: system-linux, asserts, bolt-runtime, target=aarch64{{.*}}
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t.fdata --lite=0 --hugify \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.bolt | FileCheck %s --check-prefix=HUGIFY-SECTIONS
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# HUGIFY-SECTIONS: .text PROGBITS {{[0-9a-f]+}}00000
+# HUGIFY-SECTIONS: .text.cold PROGBITS {{[0-9a-f]+}}00000
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl hot
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+.entry_hot:
+# FDATA: 1 hot #.entry_hot# 10
+ ret
+ .size hot, .-hot
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ add x0, x0, #1
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s b/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
new file mode 100644
index 0000000000000..c4cf8ce3c25be
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/injected-nonrel.s
@@ -0,0 +1,39 @@
+## Check that non-fixed injected code is allocated immediately after split cold
+## fragments in non-relocation mode and that LongJmp uses the same address.
+
+# REQUIRES: system-linux, asserts, bolt-runtime, target=aarch64{{.*}}
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/input.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t/profile.fdata --lite=0 --hugify \
+# RUN: --split-functions --split-all-cold --debug-only=longjmp \
+# RUN: 2>&1 | tee %t.log
+# RUN: llvm-nm --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
+# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment _start ends at 0x[[INJECTED:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text.injected starts at 0x[[INJECTED]]
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment __bolt_hugify_start_program starts at 0x[[INJECTED]]
+# CHECK: __bolt_hugify_start_program t [[INJECTED]]
+
+#--- input.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ cbnz x0, .Lhot
+.Lcold:
+ sub x0, x0, #1
+ ret
+.Lhot:
+ add x0, x0, #1
+ ret
+ .size _start, .-_start
+
+#--- profile.fdata
+no_lbr
+1 _start 0 10
+1 _start 4 0
+1 _start c 10
diff --git a/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s b/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
new file mode 100644
index 0000000000000..d9aaf60fef1bb
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/non-simple-nonrel.s
@@ -0,0 +1,33 @@
+## Check that non-simple functions retained at their input addresses are not
+## included in LongJmp's layout in non-relocation mode.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --print-cfg \
+# RUN: --print-only=retained --debug-only=longjmp > %t.log 2>&1
+# RUN: FileCheck %s < %t.log
+
+# CHECK: Binary Function "retained" after building cfg
+# CHECK: IsSimple : 0
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
+# CHECK-NOT: BOLT-DEBUG: LongJmp layout: main fragment retained
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ ret
+ .size _start, .-_start
+
+## The unknown indirect branch makes the function non-simple without removing
+## its CFG or instructions. In non-relocation mode,
+## shouldEmitFunctionFragment() accepts it, while BinaryContext::shouldEmit()
+## rejects it.
+ .globl retained
+ .type retained, %function
+retained:
+ br x0
+ ret
+ .size retained, .-retained
diff --git a/bolt/test/AArch64/LongJmp-Layout/sections.s b/bolt/test/AArch64/LongJmp-Layout/sections.s
new file mode 100644
index 0000000000000..012cda1bec784
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/sections.s
@@ -0,0 +1,91 @@
+## Check section-aware layout for profiled functions in .text,
+## unprofiled whole functions in .text.cold, and cold basic blocks split from a
+## hot function. The middle cold block is split only with --split-all-cold.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/input.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.unsplit.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --debug-only=longjmp 2>&1 \
+# RUN: | FileCheck %s
+# RUN: llvm-nm -n %t.unsplit.bolt | FileCheck %s --check-prefix=UNSPLIT \
+# RUN: --implicit-check-not='hot.cold.0' \
+# RUN: --implicit-check-not='trailing.cold.0'
+# RUN: llvm-bolt %t.exe -o %t.split.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-nm -n %t.split.bolt | FileCheck %s --check-prefix=SPLIT \
+# RUN: --implicit-check-not='hot.cold.0'
+# RUN: llvm-bolt %t.exe -o %t.split-all.bolt --data %t/profile.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions --split-all-cold \
+# RUN: --debug-only=longjmp 2>&1 | FileCheck %s
+# RUN: llvm-nm -n %t.split-all.bolt | FileCheck %s --check-prefix=SPLIT-ALL
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment
+# CHECK: BOLT-DEBUG: LongJmp layout: basic block
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+
+# UNSPLIT: T hot
+# UNSPLIT: T trailing
+# SPLIT: t trailing.cold.0
+# SPLIT-ALL: t hot.cold.0
+# SPLIT-ALL: t trailing.cold.0
+
+#--- input.s
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl hot
+ bl trailing
+ ret
+ .size _start, .-_start
+
+ .globl hot
+ .type hot, %function
+hot:
+ cbnz x0, .Lhot
+.Lcold_middle:
+ sub x0, x0, #1
+ b .Lexit
+.Lhot:
+ add x0, x0, #1
+.Lexit:
+ ret
+ .size hot, .-hot
+
+ .globl trailing
+ .type trailing, %function
+trailing:
+ cbz x1, .Ltrailing_cold
+ add x1, x1, #1
+ ret
+.Ltrailing_cold:
+ sub x1, x1, #1
+ ret
+ .size trailing, .-trailing
+
+ .globl cold_function
+ .type cold_function, %function
+cold_function:
+ add x2, x2, #1
+ ret
+ .size cold_function, .-cold_function
+
+ .reloc 0, R_AARCH64_NONE
+
+#--- profile.fdata
+no_lbr
+1 _start 0 10
+1 hot 0 10
+1 hot 4 0
+1 hot c 10
+1 hot 10 10
+1 trailing 0 10
+1 trailing 4 10
+1 trailing c 0
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s b/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
new file mode 100644
index 0000000000000..e8cb41a2a2ba1
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function-frontier.s
@@ -0,0 +1,84 @@
+## Check that functions marked ignored after the output function list is built
+## do not affect LongJmp's section layout. Use --force-patch to
+## make this state deterministic. Patching late_ignored fails because the
+## function is too small, so PatchEntries marks it ignored after PopulateOutputFunctions
+## has already included it as a profiled function in the output list.
+##
+## Before the patch, the frontier scan counts late_ignored, but the subsequent
+## layout loop skips it. LongJmp therefore estimates this layout:
+##
+## _start.hot | 128 MiB padding | separator | _start.cold | end
+##
+## The conditional branch from _start.hot to _start.cold appears out of range,
+## so LongJmp inserts a hot stub. The cold call to end appears to be in range,
+## so no cold stub is inserted. The emitter skips late_ignored consistently
+## while placing the fragments, and produces this layout instead:
+##
+## _start.hot | _start.cold | 128 MiB padding | separator | end
+##
+## Here, the conditional branch is in range but the cold call is out of range,
+## causing an emission failure. After the patch, the layout is built
+## from emitted fragments grouped by output section, so late_ignored is excluded.
+## The estimate then matches the emitted layout, and LongJmp inserts the required
+## cold stub instead of the unnecessary hot stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata --no-lbr %s %t.exe %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --data %t.fdata --lite=0 \
+# RUN: --reorder-functions=exec-count --split-functions --split-all-cold \
+# RUN: --skip-funcs=skipped --force-patch \
+# RUN: --pad-funcs-before=separator:134217728 2>&1 \
+# RUN: | FileCheck %s
+
+# CHECK: BOLT-WARNING: failed to patch entries in late_ignored
+# CHECK: BOLT-INFO: Inserted 0 stubs in the hot area and 1 stubs in the cold area.
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+.entry_start:
+# FDATA: 1 _start #.entry_start# 10
+ bl late_ignored
+ cbnz x0, .hot_start
+.cold_start:
+ bl end
+ ret
+.hot_start:
+# FDATA: 1 _start #.hot_start# 10
+ ret
+ .size _start, .-_start
+
+ .globl late_ignored
+ .type late_ignored, %function
+late_ignored:
+.entry_late_ignored:
+# FDATA: 1 late_ignored #.entry_late_ignored# 5
+ ret
+ .size late_ignored, .-late_ignored
+
+ .globl skipped
+ .type skipped, %function
+skipped:
+ ret
+ .size skipped, .-skipped
+
+ .globl separator
+ .type separator, %function
+separator:
+ add x0, x0, #1
+ add x0, x0, #1
+ ret
+ .size separator, .-separator
+
+ .globl end
+ .type end, %function
+end:
+ add x1, x1, #1
+ add x1, x1, #1
+ ret
+ .size end, .-end
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s b/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
new file mode 100644
index 0000000000000..a4478850cd0f8
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function-padding.s
@@ -0,0 +1,44 @@
+## Check that a skipped function and padding requested for it are both absent
+## from LongJmp's emitted-section layout.
+
+# REQUIRES: system-linux, asserts
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --lite=0 --skip-funcs=skipped \
+# RUN: --pad-funcs-before=skipped:64 --pad-funcs=skipped:64 \
+# RUN: --debug-only=longjmp > %t.log 2>&1
+# RUN: llvm-nm -n --format=posix %t.bolt >> %t.log
+# RUN: FileCheck %s < %t.log
+
+# CHECK: BOLT-DEBUG: LongJmp: layout iteration 1
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: main fragment end starts at 0x[[END:[0-9a-f]+]]
+# CHECK: BOLT-DEBUG: LongJmp layout: section .text ends at 0x
+# CHECK: _start T [[START]]
+# CHECK-NEXT: end T [[END]]
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl end
+ ret
+ .size _start, .-_start
+
+ .globl skipped
+ .type skipped, %function
+skipped:
+ add x0, x0, #1
+ ret
+ .size skipped, .-skipped
+
+ .globl end
+ .type end, %function
+end:
+ add x1, x1, #1
+ ret
+ .size end, .-end
+
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/LongJmp-Layout/skip-function.s b/bolt/test/AArch64/LongJmp-Layout/skip-function.s
new file mode 100644
index 0000000000000..321f70f2ec571
--- /dev/null
+++ b/bolt/test/AArch64/LongJmp-Layout/skip-function.s
@@ -0,0 +1,61 @@
+## Check that a skipped function does not contribute to the layout.
+## Padding puts end just within range of _start's call. The intervening target
+## function pushes end out of range unless target is skipped.
+##
+## 134217716 is 0x7fffff4. With target skipped, the displacement is:
+##
+## 0x7fffff4 (padding) + 8 (_start) = 0x7fffffc
+##
+## This is the largest positive displacement encodable by bl. When target is
+## emitted, its 8-byte size increases the displacement to 0x8000004, requiring
+## a stub.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: llvm-strip --strip-unneeded %t.exe
+# RUN: llvm-bolt %t.exe -o %t.relaxed.bolt --lite=0 \
+# RUN: --pad-funcs-before=end:134217716 2>&1 \
+# RUN: | FileCheck %s --check-prefix=RELAXED
+# RUN: llvm-objdump -d --no-show-raw-insn %t.relaxed.bolt \
+# RUN: | FileCheck %s --check-prefix=STUB-DISASM
+# RUN: llvm-bolt %t.exe -o %t.skip.bolt --lite=0 \
+# RUN: --pad-funcs-before=end:134217716 --skip-funcs=target 2>&1 \
+# RUN: | FileCheck %s --check-prefix=SKIP
+# RUN: llvm-objdump -d --no-show-raw-insn %t.skip.bolt \
+# RUN: | FileCheck %s --check-prefix=DIRECT-DISASM
+
+# RELAXED: BOLT-INFO: Inserted 1 stubs in the hot area and 0 stubs in the cold area.
+# SKIP: BOLT-INFO: Inserted 0 stubs in the hot area and 0 stubs in the cold area.
+
+# STUB-DISASM-LABEL: <_start>:
+# STUB-DISASM-NEXT: {{.*}} bl {{.*}} <_start+0x8>
+# STUB-DISASM-NEXT: {{.*}} ret
+# STUB-DISASM-NEXT: {{.*}} adrp x16,
+# STUB-DISASM-NEXT: {{.*}} add x16, x16,
+# STUB-DISASM-NEXT: {{.*}} br x16
+# DIRECT-DISASM-LABEL: <_start>:
+# DIRECT-DISASM-NEXT: {{.*}} bl {{.*}} <end>
+# DIRECT-DISASM-NEXT: {{.*}} ret
+
+ .text
+ .globl _start
+ .type _start, %function
+_start:
+ bl end
+ ret
+ .size _start, .-_start
+
+ .globl target
+ .type target, %function
+target:
+ add x0, x0, #1
+ ret
+ .size target, .-target
+
+ .globl end
+ .type end, %function
+end:
+ ret
+ .size end, .-end
diff --git a/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s b/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
index 03c35e962e99e..6353e813fadea 100644
--- a/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
+++ b/bolt/test/AArch64/long-jmp-hugify-fixup-out-of-range.s
@@ -1,4 +1,4 @@
-# The longjump pass may consider branch targets in range during tentative
+# The longjump pass may consider branch targets in range during the
# layout and decide not to insert stubs for them. Later, final section
# allocation may insert alignment padding after the last non-cold text section
# when hugify is enabled. This moves the following cold section farther away,
diff --git a/bolt/test/AArch64/split-funcs-lite.s b/bolt/test/AArch64/split-funcs-lite.s
index 5f95eea17ae75..1f70fd86ef610 100644
--- a/bolt/test/AArch64/split-funcs-lite.s
+++ b/bolt/test/AArch64/split-funcs-lite.s
@@ -1,4 +1,4 @@
-# This test checks that tentative code layout for cold blocks always runs.
+# This test checks that code layout for cold blocks always runs.
# It commonly happens when using lite mode with split functions.
# REQUIRES: system-linux, asserts
@@ -24,4 +24,4 @@ foo:
## Force relocation mode.
.reloc 0, R_AARCH64_NONE
-# CHECK: foo{{.*}} cold tentative: {{.*}}
+# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment foo{{.*}} starts at 0x
>From 9a7e44e5726cdbfb0166d06bd04fc338e1d8829c Mon Sep 17 00:00:00 2001
From: Adam Bzowski <Adam.Bzowski at arm.com>
Date: Mon, 24 Aug 2026 11:58:05 +0000
Subject: [PATCH 2/3] Rebase to apply CodeSectionOrder
Change-Id: I917ac064f12cc77c08448bb9d9ab7ee010b13284
---
bolt/lib/Core/BinaryContext.cpp | 8 ++++----
bolt/lib/Passes/LongJmp.cpp | 3 +--
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index fac31bb125f92..461240a67855c 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -108,10 +108,10 @@ namespace llvm {
namespace bolt {
CodeSectionOrder::CodeSectionOrder(const BinaryContext &BC)
- : CodeSectionOrder(
- BC.getColdCodeSectionName(), BC.getHotTextMoverSectionName(),
- BC.getMainCodeSectionName(), BC.getWarmCodeSectionName(), opts::HotText,
- opts::HotFunctionsAtEnd) {}
+ : CodeSectionOrder(BC.getColdCodeSectionName(),
+ BC.getHotTextMoverSectionName(),
+ BC.getMainCodeSectionName(), BC.getWarmCodeSectionName(),
+ opts::HotText, opts::HotFunctionsAtEnd) {}
char BOLTError::ID = 0;
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 3f383f4a83d82..a710598e00e04 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -775,10 +775,9 @@ void LongJmpPass::layout(const BinaryContext &BC,
assignFunctionsToSections(BC, SortedFunctions);
if (BC.HasRelocations) {
- const CodeSectionOrder CompareSections(BC);
-
// Mirror RewriteInstance::getCodeSections(). Sections not named explicitly
// retain their first-emission order.
+ const CodeSectionOrder CompareSections(BC);
llvm::stable_sort(
Sections, [&](const SectionPlacement &A, const SectionPlacement &B) {
return CompareSections(A.Name, B.Name);
>From 9182e95e4a501038df3eb9e1f367709dfe90782d Mon Sep 17 00:00:00 2001
From: Adam Bzowski <Adam.Bzowski at arm.com>
Date: Mon, 14 Sep 2026 18:17:18 +0000
Subject: [PATCH 3/3] Added support for warm fragments
Change-Id: I291d21730c1d3c301caa627c37b94c1798e401d3
---
bolt/lib/Passes/LongJmp.cpp | 81 +++++++++----------
bolt/test/AArch64/LongJmp-Layout/basic.s | 6 +-
.../AArch64/LongJmp-Layout/empty-function.s | 2 +-
bolt/test/AArch64/split-funcs-lite.s | 2 +-
4 files changed, 45 insertions(+), 46 deletions(-)
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index a710598e00e04..f10647ac8fded 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -394,6 +394,9 @@ void LongJmpPass::assignFunctionsToSections(
if (!shouldEmitFunctionFragment(BC, *Func))
continue;
+ assert((BC.HasRelocations || Func->getLayout().isHotColdSplit()) &&
+ "non-relocation mode supports only hot/cold splitting");
+
// RewriteInstance ultimately excludes every code section with a
// pre-assigned output address. At this point, those are the sections of
// fixed-address injected functions.
@@ -419,8 +422,6 @@ void LongJmpPass::assignFunctionsToSections(
assert((Layout.fragment_size() == 1 || Func->isSimple()) &&
"only simple functions can have multiple fragments");
for (const FunctionFragment &FF : Layout.getSplitFragments()) {
- assert(FF.getFragmentNum() == FragmentNum::cold() &&
- "LongJmp supports only main and cold function fragments");
// BinaryEmitter::emitFunctions() skips an empty split fragment unless
// the function carries a constant island.
if (FF.empty() && !Func->hasConstantIsland())
@@ -492,11 +493,9 @@ uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
bool RecordAddresses) {
assert(shouldEmitFunctionFragment(BC, Func) &&
"attempting to lay out a function BinaryEmitter will not emit");
-
const FragmentNum Fragment = FF.getFragmentNum();
- assert((Fragment == FragmentNum::main() || Fragment == FragmentNum::cold()) &&
- "LongJmp supports only main and cold function fragments");
- const bool IsCold = Fragment == FragmentNum::cold();
+ const bool IsMain = FF.isMainFragment();
+ const bool IsSplit = FF.isSplitFragment();
const bool HasFixedOutputAddress =
Func.isInjected() && Func.getOutputAddress();
@@ -505,7 +504,7 @@ uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
const bool NeedsNonRelocInjectedAlignment =
!BC.HasRelocations && Func.isInjected() && !HasFixedOutputAddress;
const bool NeedsNonRelocColdAlignment =
- !BC.HasRelocations && !Func.isInjected() && IsCold;
+ !BC.HasRelocations && !Func.isInjected() && IsSplit;
// Apply the alignment that affects the fragment's mapped address.
// Section-relative fragments mirror BinaryEmitter::emitFunction(); moved
@@ -515,7 +514,7 @@ uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
if (NeedsRelocationAlignment) {
DotAddress = alignTo(DotAddress, Func.getMinAlignment());
const uint16_t MaxAlignmentBytes =
- IsCold ? Func.getMaxColdAlignmentBytes() : Func.getMaxAlignmentBytes();
+ IsSplit ? Func.getMaxColdAlignmentBytes() : Func.getMaxAlignmentBytes();
if (MaxAlignmentBytes > 0)
DotAddress =
applyCodeAlignment(DotAddress, Func.getAlign(), MaxAlignmentBytes);
@@ -535,14 +534,8 @@ uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
DotAddress += opts::padFunctionBefore(Func);
// BinaryEmitter::emitFunction() emits the fragment entry symbols here.
- if (RecordAddresses) {
- if (!IsCold)
- HotAddresses[&Func] = DotAddress;
- LLVM_DEBUG(dbgs() << "BOLT-DEBUG: LongJmp layout: "
- << (IsCold ? "cold" : "main") << " fragment "
- << Func.getPrintName() << " starts at 0x"
- << Twine::utohexstr(DotAddress) << '\n');
- }
+ if (RecordAddresses && IsMain)
+ HotAddresses[&Func] = DotAddress;
// --break-funcs emits UD2 before the function body.
DotAddress += opts::breakFunctionSize(Func);
@@ -557,10 +550,19 @@ uint64_t LongJmpPass::layoutFunctionFragment(const BinaryContext &BC,
DotAddress += opts::markFunctionBytes(BC).size();
LLVM_DEBUG({
- if (RecordAddresses)
- dbgs() << "BOLT-DEBUG: LongJmp layout: " << (IsCold ? "cold" : "main")
- << " fragment " << Func.getPrintName() << " ends at 0x"
- << Twine::utohexstr(DotAddress) << '\n';
+ if (RecordAddresses) {
+ const StringRef FragmentName =
+ IsMain ? "main"
+ : BC.HasWarmSection && Fragment == FragmentNum::warm() ? "warm"
+ : Fragment == FragmentNum::cold() ? "cold"
+ : "split";
+ dbgs() << "BOLT-DEBUG: LongJmp layout: " << FragmentName << " fragment "
+ << Func.getPrintName();
+ if (IsMain)
+ dbgs() << " starts at 0x" << Twine::utohexstr(HotAddresses.at(&Func))
+ << " and";
+ dbgs() << " ends at 0x" << Twine::utohexstr(DotAddress) << '\n';
+ }
});
return DotAddress;
@@ -597,35 +599,39 @@ uint64_t LongJmpPass::layoutSection(const BinaryContext &BC,
uint64_t LongJmpPass::layoutSectionsForward(const BinaryContext &BC,
uint64_t DotAddress) {
- const bool AdjustMainSection =
+ const StringRef LastNonColdSectionName = BC.HasWarmSection
+ ? BC.getWarmCodeSectionName()
+ : BC.getMainCodeSectionName();
+ const bool AdjustLastNonColdSection =
BC.HasRelocations &&
(opts::HotText || (opts::Hugify && !BC.HasFixedLoadAddress));
- std::optional<uint64_t> MainSectionEnd = std::nullopt;
+ std::optional<uint64_t> LastNonColdSectionEnd = std::nullopt;
// Mirror allocateAt() in RewriteInstance::mapCodeSections().
for (const SectionPlacement &Section : Sections) {
DotAddress = alignTo(DotAddress, Section.Alignment);
DotAddress = layoutSection(BC, Section, DotAddress);
- if (AdjustMainSection &&
- StringRef(Section.Name) == BC.getMainCodeSectionName()) {
+ if (AdjustLastNonColdSection &&
+ StringRef(Section.Name) == LastNonColdSectionName) {
if (opts::HotText)
- MainSectionEnd = DotAddress;
- // LongJmp supports only main and cold fragments. Mirror the extra
- // post-main alignment in allocateAt() for --hugify.
+ LastNonColdSectionEnd = DotAddress;
+ // Mirror the extra post-hot-text alignment in allocateAt() for
+ // --hugify. With CDSplit, warm code is part of hot text.
if (opts::Hugify && !BC.HasFixedLoadAddress)
DotAddress = alignTo(DotAddress, Section.Alignment);
}
}
// Mirror RewriteInstance::mapCodeSections() padding used to accommodate
- // hot-text huge-page mapping. LongJmp does not support warm fragments, so the
- // hot-text end is the end of the main section. RewriteInstance applies this
- // adjustment only in allocateAt() to advance the next free address;
- // allocateBefore() starts from a fixed upper boundary and has no
- // corresponding adjustment.
- if (MainSectionEnd)
- DotAddress = std::max(DotAddress, alignTo(*MainSectionEnd, BC.PageAlign));
+ // hot-text huge-page mapping. The hot-text end is the end of the warm
+ // section when one exists, and the end of the main section otherwise.
+ // RewriteInstance applies this adjustment only in allocateAt() to advance
+ // the next free address; allocateBefore() starts from a fixed upper boundary
+ // and has no corresponding adjustment.
+ if (LastNonColdSectionEnd)
+ DotAddress =
+ std::max(DotAddress, alignTo(*LastNonColdSectionEnd, BC.PageAlign));
return DotAddress;
}
@@ -714,8 +720,6 @@ void LongJmpPass::layoutFunctions(
Func.getAddress());
if (Func.isSplit()) {
- assert(Func.getLayout().isHotColdSplit() &&
- "non-relocation mode supports only hot/cold splitting");
ColdAddress = layoutFunctionFragment(
BC, Func, Func.getLayout().getFragment(FragmentNum::cold()),
ColdAddress);
@@ -1651,11 +1655,6 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
}
Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
-
- assert((opts::CompactCodeModel || opts::ExperimentalRelaxation ||
- opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) &&
- "LongJmp cannot work with functions split in more than two fragments");
-
DenseMap<BinaryFunction *, BranchLivenessInfo> BranchLiveness;
if (opts::FixBranchesWithLiveness) {
diff --git a/bolt/test/AArch64/LongJmp-Layout/basic.s b/bolt/test/AArch64/LongJmp-Layout/basic.s
index 1eeeb59841c51..14dfec8d235a7 100644
--- a/bolt/test/AArch64/LongJmp-Layout/basic.s
+++ b/bolt/test/AArch64/LongJmp-Layout/basic.s
@@ -44,8 +44,8 @@
# ALIGN: LongJmp layout: section .text starts at 0x[[TEXT:[0-9a-f]+]], alignment 0x1000
# ALIGN: LongJmp layout: main fragment _start starts at 0x[[START:[0-9a-f]+]]
-# ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
# ALIGN: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
+# ALIGN: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
# ALIGN: LongJmp layout: main fragment second starts at 0x[[SECOND:[0-9a-f]+]]
# ALIGN: _start T [[START]]
# ALIGN: first T [[FIRST]]
@@ -67,10 +67,10 @@
# NONRELOC: BOLT-WARNING: non-relocation mode for AArch64 is not fully supported
# NONRELOC: BOLT-DEBUG: LongJmp layout starts at 0x
-# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment first starts at 0x[[FIRST:[0-9a-f]+]]
-# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x[[FIRST]]
+# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x[[FIRST:[0-9a-f]+]]
# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]+}}
# NONRELOC: BOLT-DEBUG: LongJmp layout: basic block {{.*}} in first starts at 0x{{[0-9a-f]*[02468ace]0}}
+# NONRELOC: BOLT-DEBUG: LongJmp layout: main fragment first starts at 0x[[FIRST]]
.text
.p2align 6
diff --git a/bolt/test/AArch64/LongJmp-Layout/empty-function.s b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
index 8591a8410bf1d..26fa123a139fe 100644
--- a/bolt/test/AArch64/LongJmp-Layout/empty-function.s
+++ b/bolt/test/AArch64/LongJmp-Layout/empty-function.s
@@ -43,7 +43,7 @@
# RELOC: BOLT-DEBUG: LongJmp layout: main fragment _start starts at 0x
# RELOC: BOLT-DEBUG: LongJmp layout: main fragment next starts at 0x
# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only starts at 0x
-# RELOC: BOLT-DEBUG: LongJmp layout: main fragment nop_only ends at 0x
+# RELOC-SAME: and ends at 0x
# MAIN-SYMBOL: T nop_only
diff --git a/bolt/test/AArch64/split-funcs-lite.s b/bolt/test/AArch64/split-funcs-lite.s
index 1f70fd86ef610..bd251e26e02b7 100644
--- a/bolt/test/AArch64/split-funcs-lite.s
+++ b/bolt/test/AArch64/split-funcs-lite.s
@@ -24,4 +24,4 @@ foo:
## Force relocation mode.
.reloc 0, R_AARCH64_NONE
-# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment foo{{.*}} starts at 0x
+# CHECK: BOLT-DEBUG: LongJmp layout: cold fragment foo{{.*}} ends at 0x
More information about the llvm-commits
mailing list