[llvm] [BOLT] Enable relocation recovery for stripped ELF binaries (PR #221991)
Wenlong Mu via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 06:10:51 PDT 2026
https://github.com/onroadmuwl created https://github.com/llvm/llvm-project/pull/221991
Add opt-in relocation recovery for stripped AArch64 and x86-64 ELF binaries that lack the static relocation information normally required by BOLT's relocation-mode rewriting.
The `--recover-relocations` option enables relocation recovery when the required static relocation information is unavailable. The `--aggressive-relocation-recovery` option additionally recovers function pointers from general data sections and requires `--recover-relocations` to be enabled. These options are opt-in and do not affect BOLT's existing rewriting mode for binaries that retain symbol tables and static relocation information.
The implementation recovers code and data references required during binary rewriting and updates them after code layout changes. It also supports relocating jump tables together with optimized code and updating the corresponding references.
On AArch64, the implementation recovers ADRP/ADD address references and function pointers from supported data sections. When the complete target of an ADRP/ADD reference cannot be determined unambiguously, the original ADRP page is preserved while the ADD immediate remains unchanged.
For stripped AArch64 binaries, the patch also recovers additional functions that may not be discoverable from the remaining binary metadata, including the process entry point (`_start`), its optional glibc main-wrapper secondary entry, and Cortex-A53 erratum 843419 veneers.
Regression tests are added for both AArch64 and x86-64 to validate relocation recovery and the correctness of rewritten addresses.
The implementation has also been evaluated on SPEC CPU2017 and real-world applications including MySQL and MongoDB. The optimized stripped binaries achieve performance results close to those produced by BOLT on binaries retaining symbol tables and static relocation information.
>From d94efa5a30a7c69cedefe101b597c5892bef0bf6 Mon Sep 17 00:00:00 2001
From: Wenlong Mu <muwl182 at 163.com>
Date: Tue, 8 Sep 2026 20:06:00 +0800
Subject: [PATCH] [BOLT] Add relocation recovery for stripped ELF binaries
Add opt-in relocation recovery for AArch64 and x86-64 ELF inputs
without the required static relocation records.
Recover AArch64 ADRP/ADD references and function pointers in supported
data sections, preserve the original ADRP page when the complete target
is ambiguous, and move jump tables with optimized code. For stripped
AArch64 inputs, discover _start, its optional main-wrapper secondary
entry, and Cortex-A53 erratum 843419 veneers.
Add tests that validate recovered addresses directly in the output ELF.
---
bolt/include/bolt/Core/BinaryContext.h | 7 +-
bolt/include/bolt/Passes/RelocationRecovery.h | 33 ++
bolt/include/bolt/Rewrite/RewriteInstance.h | 6 +-
bolt/lib/Core/BinaryContext.cpp | 81 ++--
bolt/lib/Core/BinaryFunction.cpp | 6 +-
bolt/lib/Passes/CMakeLists.txt | 1 +
bolt/lib/Passes/RelocationRecovery.cpp | 361 ++++++++++++++++++
bolt/lib/Rewrite/BinaryPassManager.cpp | 10 +
bolt/lib/Rewrite/CMakeLists.txt | 1 +
bolt/lib/Rewrite/RewriteInstance.cpp | 77 +++-
bolt/lib/Rewrite/StrippedBinary.cpp | 285 ++++++++++++++
.../AArch64/recover-relocations-ambiguous.s | 33 ++
bolt/test/AArch64/recover-relocations.s | 92 +++++
bolt/test/AArch64/stripped-entry.s | 52 +++
bolt/test/AArch64/stripped-erratum-843419.s | 57 +++
bolt/test/Inputs/check-recovered-addresses.py | 75 ++++
bolt/test/X86/recover-relocations-data.s | 45 +++
bolt/test/X86/stripped-jump-table.s | 49 +++
.../gn/secondary/bolt/lib/Passes/BUILD.gn | 1 +
.../gn/secondary/bolt/lib/Rewrite/BUILD.gn | 1 +
20 files changed, 1238 insertions(+), 35 deletions(-)
create mode 100644 bolt/include/bolt/Passes/RelocationRecovery.h
create mode 100644 bolt/lib/Passes/RelocationRecovery.cpp
create mode 100644 bolt/lib/Rewrite/StrippedBinary.cpp
create mode 100644 bolt/test/AArch64/recover-relocations-ambiguous.s
create mode 100644 bolt/test/AArch64/recover-relocations.s
create mode 100644 bolt/test/AArch64/stripped-entry.s
create mode 100644 bolt/test/AArch64/stripped-erratum-843419.s
create mode 100644 bolt/test/Inputs/check-recovered-addresses.py
create mode 100644 bolt/test/X86/recover-relocations-data.s
create mode 100644 bolt/test/X86/stripped-jump-table.s
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 2ccccde342e7f..f50d25add3894 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -767,9 +767,14 @@ class BinaryContext {
/// Indicates if the binary is Linux kernel.
bool IsLinuxKernel{false};
- /// Indicates if relocations are available for usage.
+ /// Indicates whether the relocation-mode rewriting pipeline is active.
bool HasRelocations{false};
+ /// True when --recover-relocations is enabled. BOLT reconstructs code and
+ /// data references that are missing from the input static relocation records,
+ /// then uses relocation-mode rewriting to move the referenced functions.
+ bool RecoverRelocations{false};
+
/// Indicates if the binary is stripped
bool IsStripped{false};
diff --git a/bolt/include/bolt/Passes/RelocationRecovery.h b/bolt/include/bolt/Passes/RelocationRecovery.h
new file mode 100644
index 0000000000000..047acaffede08
--- /dev/null
+++ b/bolt/include/bolt/Passes/RelocationRecovery.h
@@ -0,0 +1,33 @@
+//===- bolt/Passes/RelocationRecovery.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_RELOCATIONRECOVERY_H
+#define BOLT_PASSES_RELOCATIONRECOVERY_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+/// Reconstruct code and data address references that are missing from the
+/// input static relocation records. If the complete target of an AArch64
+/// ADRP/ADD reference cannot be determined unambiguously, rewrite the ADRP to
+/// reproduce its original absolute page and leave the ADD immediate unchanged.
+class RelocationRecovery : public BinaryFunctionPass {
+public:
+ explicit RelocationRecovery(const cl::opt<bool> &PrintPass)
+ : BinaryFunctionPass(PrintPass) {}
+
+ const char *getName() const override { return "recover-relocations"; }
+ Error runOnFunctions(BinaryContext &BC) override;
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif // BOLT_PASSES_RELOCATIONRECOVERY_H
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 4d4ac70195bc2..4c1028b723f11 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -87,7 +87,7 @@ class RewriteInstance {
private:
/// Populate array of binary functions and other objects of interest
/// from meta data in the file.
- void discoverFileObjects();
+ Error discoverFileObjects();
/// Check if the input binary has a space reserved for BOLT and use it for new
/// section allocations if found.
@@ -175,6 +175,10 @@ class RewriteInstance {
/// optimization.
void disassembleFunctions();
+ /// Discover AArch64 process-entry code and linker-generated veneers in a
+ /// stripped input.
+ Error discoverStrippedFunctions();
+
void buildFunctionsCFG();
void postProcessFunctions();
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index c40851d9aaa98..296cd3b7ca7dd 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -635,6 +635,8 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
const uint64_t NextJTAddress,
JumpTable::AddressesType *EntriesAsAddress,
bool *HasEntryInFragment) const {
+ const bool HasSymbolTable = !IsStripped;
+
// Target address of __builtin_unreachable.
const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();
@@ -701,7 +703,7 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
LLVM_DEBUG(dbgs() << " * Checking 0x" << Twine::utohexstr(EntryAddress)
<< " -> ");
// Check if there's a proper relocation against the jump table entry.
- if (HasRelocations) {
+ if (HasRelocations && !RecoverRelocations) {
if (Type == JumpTable::JTT_PIC &&
!DataPCRelocations.count(EntryAddress)) {
LLVM_DEBUG(
@@ -740,7 +742,16 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
// Function or one of its fragments.
const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);
- if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) {
+ if (!TargetBF) {
+ LLVM_DEBUG(printEntryDiagnostics(dbgs(), TargetBF));
+ (void)printEntryDiagnostics;
+ break;
+ }
+
+ // areRelatedFragments() identifies split-function fragments by symbol name.
+ // A stripped binary has no names for this check, so skip it after verifying
+ // above that Value lies within a BinaryFunction registered in the context.
+ if (HasSymbolTable && !areRelatedFragments(TargetBF, &BF)) {
LLVM_DEBUG(printEntryDiagnostics(dbgs(), TargetBF));
(void)printEntryDiagnostics;
break;
@@ -934,20 +945,28 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
if (llvm::is_contained(JT->Parents, &Function))
return JT->getFirstLabel();
- // Prevent associating a jump table to a specific fragment twice.
- auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this,
- &Function, std::placeholders::_1);
- assert(llvm::all_of(JT->Parents, isSibling) &&
- "cannot reuse jump table of a different function");
- (void)isSibling;
+ // Symbol names establish fragment relationships. Without a symbol table,
+ // separately discovered functions can refer to the same jump table but
+ // cannot be classified as fragments by name.
+ if (!IsStripped) {
+ auto IsSibling = std::bind(&BinaryContext::areRelatedFragments, this,
+ &Function, std::placeholders::_1);
+ assert(llvm::all_of(JT->Parents, IsSibling) &&
+ "cannot reuse jump table of a different function");
+ (void)IsSibling;
+ }
if (opts::Verbosity > 2) {
- this->outs() << "BOLT-INFO: multiple fragments access the same jump table"
- << ": " << *JT->Parents[0] << "; " << Function << '\n';
+ this->outs() << "BOLT-INFO: multiple "
+ << (IsStripped ? "functions" : "fragments")
+ << " access the same jump table: " << *JT->Parents[0] << "; "
+ << Function << '\n';
JT->print(this->outs());
}
- if (JT->Parents.size() == 1)
- JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);
- Function.setHasIndirectTargetToSplitFragment(true);
+ if (!IsStripped) {
+ if (JT->Parents.size() == 1)
+ JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);
+ Function.setHasIndirectTargetToSplitFragment(true);
+ }
// Duplicate the entry for the parent function for easy access
JT->Parents.push_back(&Function);
Function.JumpTables.emplace(Address, JT);
@@ -1093,20 +1112,32 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
auto isNoop = std::bind(&MCPlusBuilder::isNoop, MIB.get(), _1);
- // Some functions have a jump to the next function or to the padding area
- // inserted after the body.
+ // Some functions have a jump over linker-inserted code or padding after the
+ // body. On AArch64, accept the skipped range only when it contains an
+ // erratum 843419 helper discovered earlier.
auto isSkipJump = [&](const MCInst &Instr) {
- if (!isX86())
- return false;
uint64_t TargetAddress = 0;
- if (MIB->isUnconditionalBranch(Instr) &&
- MIB->evaluateBranch(Instr, InstrAddress, InstrSize, TargetAddress)) {
- if (TargetAddress >= InstrAddress + InstrSize &&
- TargetAddress <= BF.getAddress() + BF.getMaxSize()) {
- return true;
- }
- }
- return false;
+ if (!MIB->isUnconditionalBranch(Instr) ||
+ !MIB->evaluateBranch(Instr, InstrAddress, InstrSize, TargetAddress) ||
+ TargetAddress < InstrAddress + InstrSize)
+ return false;
+
+ if (isX86())
+ return TargetAddress <= BF.getAddress() + BF.getMaxSize();
+ if (!isAArch64() || !RecoverRelocations)
+ return false;
+
+ auto TargetSection = getSectionForAddress(TargetAddress);
+ if (!TargetSection || &*TargetSection != BF.getOriginSection())
+ return false;
+
+ // GNU ld places a forward branch before an erratum veneer island. Stripped
+ // function discovery registers the veneer before padding validation, so a
+ // matching function in the skipped range identifies this layout.
+ auto NextFunction = BinaryFunctions.upper_bound(InstrAddress);
+ return NextFunction != BinaryFunctions.end() &&
+ NextFunction->first < TargetAddress &&
+ NextFunction->second.getOneName().starts_with("__BOLT_e843419_");
};
// For veneers that are not already covered by binary functions, only those
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 99d6d27264fbe..a9f7e6f6e995a 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -2329,6 +2329,9 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
const uint32_t Offset = I->first;
MCInst &Instr = I->second;
+ if (BC.RecoverRelocations && BC.isAArch64() && BC.MIB->isADRP(Instr))
+ MIB->setOffset(Instr, Offset);
+
auto LI = Labels.find(Offset);
if (LI != Labels.end()) {
// Always create new BB at branch destination.
@@ -2570,7 +2573,8 @@ void BinaryFunction::postProcessCFG() {
if (!requiresPreciseAddressMap() && !opts::Instrument) {
for (BinaryBasicBlock &BB : blocks())
for (MCInst &Inst : BB)
- BC.MIB->clearOffset(Inst);
+ if (!(BC.RecoverRelocations && BC.isAArch64() && BC.MIB->isADRP(Inst)))
+ BC.MIB->clearOffset(Inst);
}
assert((!isSimple() || validateCFG()) &&
diff --git a/bolt/lib/Passes/CMakeLists.txt b/bolt/lib/Passes/CMakeLists.txt
index 686dee6987a73..65059ad90435d 100644
--- a/bolt/lib/Passes/CMakeLists.txt
+++ b/bolt/lib/Passes/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_library(LLVMBOLTPasses
DataflowInfoManager.cpp
FrameAnalysis.cpp
FrameOptimizer.cpp
+ RelocationRecovery.cpp
FixRelaxationPass.cpp
FixRISCVCallsPass.cpp
HFSort.cpp
diff --git a/bolt/lib/Passes/RelocationRecovery.cpp b/bolt/lib/Passes/RelocationRecovery.cpp
new file mode 100644
index 0000000000000..6cb1dfbbf7f22
--- /dev/null
+++ b/bolt/lib/Passes/RelocationRecovery.cpp
@@ -0,0 +1,361 @@
+//===- bolt/Passes/RelocationRecovery.cpp -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Passes/RelocationRecovery.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/DataExtractor.h"
+
+using namespace llvm;
+
+namespace opts {
+extern cl::OptionCategory BoltCategory;
+extern cl::opt<bool> Instrument;
+cl::opt<bool> AggressiveRelocationRecovery(
+ "aggressive-relocation-recovery",
+ cl::desc("also recover function pointers from general data sections; "
+ "requires --recover-relocations"),
+ cl::Hidden, cl::cat(BoltCategory));
+} // namespace opts
+
+namespace llvm {
+namespace bolt {
+namespace {
+
+/// Information collected for one ADRP instruction before rewriting it.
+/// Page is the absolute page computed by the original ADRP. Adds contains the
+/// ADD instructions that use this ADRP result to form the same target address;
+/// Symbol names that target when it can be recovered unambiguously.
+struct AdrpDefinition {
+ MCInst *ADRP;
+ uint64_t Page;
+ MCSymbol *Symbol = nullptr;
+ SmallVector<MCInst *, 4> Adds;
+ bool ClearOffsetAfterRecovery;
+};
+
+Error recoveryError(const BinaryFunction &BF, StringRef Reason) {
+ return createFatalBOLTError(Twine("cannot recover references in ") +
+ BF.getPrintName() + ": " + Reason);
+}
+
+/// Follow an ADRP definition through the CFG and collect ADD uses that compute
+/// one target. Record all definitions and uses before rewriting operands
+/// because multiple definitions can reach the same ADD. When the target is not
+/// unique, preserve the input page in the ADRP and leave the ADD unchanged.
+Error collectAdrpDefUseChains(
+ BinaryFunction &BF, SmallVectorImpl<AdrpDefinition> &Definitions,
+ DenseMap<MCInst *, BinaryBasicBlock *> &AddBlocks) {
+ BinaryContext &BC = BF.getBinaryContext();
+ for (BinaryBasicBlock &BB : BF) {
+ for (auto I = BB.begin(); I != BB.end(); ++I) {
+ MCInst &ADRP = *I;
+ if (!BC.MIB->isADRP(ADRP) || !ADRP.getOperand(1).isImm())
+ continue;
+ const std::optional<uint32_t> Offset = BC.MIB->getOffset(ADRP);
+ if (!Offset)
+ return recoveryError(BF, "missing ADRP input offset");
+ const uint64_t Page = ((BF.getAddress() + *Offset) & ~uint64_t(4095)) +
+ uint64_t(ADRP.getOperand(1).getImm()) * 4096;
+ AdrpDefinition Definition{&ADRP,
+ Page,
+ nullptr,
+ {},
+ !BF.requiresPreciseAddressMap() &&
+ !opts::Instrument};
+ const unsigned Reg = ADRP.getOperand(0).getReg();
+ SmallVector<BinaryBasicBlock *, 8> Worklist;
+ SmallPtrSet<BinaryBasicBlock *, 8> Visited;
+ Worklist.push_back(&BB);
+ bool First = true;
+ bool Complete = true;
+ std::optional<uint64_t> Target;
+ while (!Worklist.empty()) {
+ BinaryBasicBlock *Current = Worklist.pop_back_val();
+ auto Begin = First ? std::next(I) : Current->begin();
+ const bool IsInitial = First;
+ First = false;
+ if (!IsInitial && !Visited.insert(Current).second)
+ continue;
+ bool Killed = false;
+ for (auto II = Begin; II != Current->end(); ++II) {
+ MCInst &Inst = *II;
+ if (BC.MIB->isPseudo(Inst))
+ continue;
+ if (BC.MIB->isCall(Inst)) {
+ Complete = false;
+ Killed = true;
+ break;
+ }
+ const bool Uses = BC.MIB->hasUseOfPhysReg(Inst, Reg);
+ const bool Defines = BC.MIB->hasDefOfPhysReg(Inst, Reg);
+ if (Uses) {
+ if (!BC.MIB->isAddXri(Inst) || Inst.getOperand(1).getReg() != Reg ||
+ !Inst.getOperand(2).isImm() || !Inst.getOperand(3).isImm() ||
+ Inst.getOperand(3).getImm()) {
+ Complete = false;
+ } else {
+ const uint64_t Address = Page + Inst.getOperand(2).getImm();
+ if (Target && *Target != Address)
+ Complete = false;
+ Target = Address;
+ Definition.Adds.push_back(&Inst);
+ AddBlocks.try_emplace(&Inst, Current);
+ }
+ }
+ if (Defines) {
+ Killed = true;
+ break;
+ }
+ }
+ if (!Killed)
+ for (BinaryBasicBlock *Succ : Current->successors())
+ Worklist.push_back(Succ);
+ }
+ if (Target) {
+ if (BinaryFunction *Dest = BC.getBinaryFunctionAtAddress(*Target)) {
+ Definition.Symbol = Dest->getSymbol();
+ } else if (BinaryFunction *Dest =
+ BC.getBinaryFunctionContainingAddress(*Target)) {
+ if (!Dest->isInConstantIsland(*Target)) {
+ if (const BinaryBasicBlock *Entry =
+ Dest->getBasicBlockAtOffset(*Target - Dest->getAddress()))
+ Definition.Symbol = Dest->getSecondaryEntryPointSymbol(*Entry);
+ }
+ } else if (BC.getJumpTableContainingAddress(*Target)) {
+ Definition.Symbol =
+ BC.getOrCreateGlobalSymbol(*Target, "JUMP_TABLE/");
+ }
+ }
+ if (!Complete)
+ Definition.Symbol = nullptr;
+ Definitions.push_back(std::move(Definition));
+ }
+ }
+ return Error::success();
+}
+
+/// Walk backward from Use through its predecessor blocks and find the
+/// definition of Reg on every path. Succeed only when each path reaches an ADRP
+/// before a call, a non-ADRP definition of Reg, or a function-entry boundary.
+/// Scan UseBlock only before Use; if a loop reaches it again, scan the complete
+/// block. Each complete predecessor block is visited at most once.
+bool collectReachingAdrpDefinitions(
+ BinaryContext &BC, BinaryBasicBlock &UseBlock, MCInst &Use, unsigned Reg,
+ SmallPtrSetImpl<MCInst *> &ReachingDefinitions) {
+ struct BlockPosition {
+ BinaryBasicBlock *Block;
+ MCInst *Stop;
+ };
+ SmallVector<BlockPosition, 8> Worklist{{&UseBlock, &Use}};
+ SmallPtrSet<BinaryBasicBlock *, 8> VisitedFullBlocks;
+
+ while (!Worklist.empty()) {
+ const BlockPosition Position = Worklist.pop_back_val();
+ BinaryBasicBlock *Block = Position.Block;
+ auto End = Block->end();
+ if (Position.Stop) {
+ End = llvm::find_if(*Block,
+ [&](MCInst &Inst) { return &Inst == Position.Stop; });
+ if (End == Block->end())
+ return false;
+ } else if (!VisitedFullBlocks.insert(Block).second) {
+ continue;
+ }
+
+ bool FoundDefinition = false;
+ for (auto I = std::make_reverse_iterator(End), E = Block->rend(); I != E;
+ ++I) {
+ MCInst &Inst = *I;
+ if (BC.MIB->isPseudo(Inst))
+ continue;
+ if (BC.MIB->isCall(Inst))
+ return false;
+ if (!BC.MIB->hasDefOfPhysReg(Inst, Reg))
+ continue;
+ if (!BC.MIB->isADRP(Inst) || Inst.getOperand(0).getReg() != Reg ||
+ !Inst.getOperand(1).isImm())
+ return false;
+ ReachingDefinitions.insert(&Inst);
+ FoundDefinition = true;
+ break;
+ }
+ if (FoundDefinition)
+ continue;
+ if (Block->isEntryPoint() || Block->isLandingPad() || Block->pred_empty())
+ return false;
+ for (BinaryBasicBlock *Pred : Block->predecessors())
+ Worklist.push_back({Pred, nullptr});
+ }
+ return !ReachingDefinitions.empty();
+}
+
+Error recoverAArch64InstructionReferences(BinaryContext &BC) {
+ SmallVector<AdrpDefinition, 16> Definitions;
+ DenseMap<MCInst *, BinaryBasicBlock *> AddBlocks;
+ for (auto &BFI : BC.getBinaryFunctions()) {
+ BinaryFunction &BF = BFI.second;
+ if (!BC.shouldEmit(BF))
+ continue;
+ if (Error E = collectAdrpDefUseChains(BF, Definitions, AddBlocks))
+ return E;
+ }
+
+ DenseMap<MCInst *, AdrpDefinition *> DefinitionsByInstruction;
+ SmallPtrSet<MCInst *, 16> CandidateAdds;
+ for (AdrpDefinition &Definition : Definitions) {
+ DefinitionsByInstruction.try_emplace(Definition.ADRP, &Definition);
+ CandidateAdds.insert_range(Definition.Adds);
+ }
+
+ DenseMap<MCInst *, MCSymbol *> CandidateAddTargets;
+ SmallPtrSet<MCInst *, 16> UnsafeAdds;
+ for (MCInst *Add : CandidateAdds) {
+ auto BlockIt = AddBlocks.find(Add);
+ if (BlockIt == AddBlocks.end()) {
+ UnsafeAdds.insert(Add);
+ continue;
+ }
+
+ const unsigned Reg = Add->getOperand(1).getReg();
+ SmallPtrSet<MCInst *, 4> ReachingDefinitions;
+ if (!collectReachingAdrpDefinitions(BC, *BlockIt->second, *Add, Reg,
+ ReachingDefinitions)) {
+ UnsafeAdds.insert(Add);
+ continue;
+ }
+
+ std::optional<uint64_t> Page;
+ MCSymbol *Symbol = nullptr;
+ bool Valid = true;
+ for (MCInst *ADRP : ReachingDefinitions) {
+ auto DefinitionIt = DefinitionsByInstruction.find(ADRP);
+ if (DefinitionIt == DefinitionsByInstruction.end() ||
+ !DefinitionIt->second->Symbol ||
+ !llvm::is_contained(DefinitionIt->second->Adds, Add)) {
+ Valid = false;
+ break;
+ }
+ const AdrpDefinition &Definition = *DefinitionIt->second;
+ if ((Page && *Page != Definition.Page) ||
+ (Symbol && Symbol != Definition.Symbol)) {
+ Valid = false;
+ break;
+ }
+ Page = Definition.Page;
+ Symbol = Definition.Symbol;
+ }
+ if (!Valid || !Symbol)
+ UnsafeAdds.insert(Add);
+ else
+ CandidateAddTargets.try_emplace(Add, Symbol);
+ }
+
+ // An ADRP may feed several ADDs, and an ADD may be reached by several
+ // ADRPs. Their rewrites therefore cannot be decided independently. If one
+ // ADD is unsafe, make every ADRP that can reach it fall back to its original
+ // page and mark every other ADD fed by those ADRPs as unsafe. Repeat until
+ // no additional ADRP or ADD becomes unsafe, then rewrite the remaining pairs.
+ bool Changed;
+ do {
+ Changed = false;
+ for (AdrpDefinition &Definition : Definitions) {
+ if (!Definition.Symbol ||
+ !llvm::any_of(Definition.Adds,
+ [&](MCInst *Add) { return UnsafeAdds.contains(Add); }))
+ continue;
+ Definition.Symbol = nullptr;
+ for (MCInst *Add : Definition.Adds)
+ Changed |= UnsafeAdds.insert(Add).second;
+ }
+ } while (Changed);
+
+ DenseMap<MCInst *, MCSymbol *> AddTargets;
+ for (const auto &[Add, Symbol] : CandidateAddTargets)
+ if (!UnsafeAdds.contains(Add))
+ AddTargets.try_emplace(Add, Symbol);
+
+ for (AdrpDefinition &Definition : Definitions) {
+ // Without a recovered pair target, encode the absolute input page rather
+ // than retaining a PC-relative immediate whose meaning changes when the
+ // instruction moves.
+ MCSymbol *Symbol = Definition.Symbol;
+ if (!Symbol)
+ Symbol = BC.registerNameAtAddress("__BOLT_zero_addr", 0, 0, 0);
+ int64_t Value;
+ BC.MIB->replaceImmWithSymbolRef(
+ *Definition.ADRP, Symbol, Definition.Symbol ? 0 : Definition.Page,
+ BC.Ctx.get(), Value, ELF::R_AARCH64_ADR_PREL_PG_HI21);
+ if (Definition.ClearOffsetAfterRecovery)
+ BC.MIB->clearOffset(*Definition.ADRP);
+ }
+ for (const auto &[Add, Symbol] : AddTargets) {
+ if (!Symbol)
+ continue;
+ int64_t Value;
+ BC.MIB->replaceImmWithSymbolRef(*Add, Symbol, 0, BC.Ctx.get(), Value,
+ ELF::R_AARCH64_ADD_ABS_LO12_NC);
+ }
+ return Error::success();
+}
+
+/// Reconstruct absolute relocations for aligned words that exactly equal a
+/// movable function entry. Conservative mode scans ELF pointer arrays;
+/// aggressive mode also scans selected general data sections.
+Error reconstructDataRelocations(BinaryContext &BC) {
+ struct PointerReference {
+ uint64_t Address;
+ uint64_t Target;
+ MCSymbol *Symbol;
+ };
+ SmallVector<PointerReference, 16> References;
+ for (BinarySection &Section : BC.sections()) {
+ const StringRef Name = Section.getName();
+ const bool IsArray = Name == ".init_array" || Name == ".fini_array";
+ const bool IsAggressiveSection = Name == ".data.rel.ro" ||
+ Name == ".data" || Name == ".rodata" ||
+ Name == ".tdata";
+ if (!IsArray &&
+ (!opts::AggressiveRelocationRecovery || !IsAggressiveSection))
+ continue;
+ DataExtractor Data(Section.getContents(), BC.AsmInfo->isLittleEndian());
+ uint64_t Offset = (8 - Section.getAddress() % 8) % 8;
+ while (Data.isValidOffsetForDataOfSize(Offset, 8)) {
+ const uint64_t Address = Section.getAddress() + Offset;
+ const uint64_t Target = Data.getU64(&Offset);
+ if (BC.getDynamicRelocationAt(Address) ||
+ Section.getRelocationAt(Address - Section.getAddress()))
+ continue;
+ BinaryFunction *BF = BC.getBinaryFunctionAtAddress(Target);
+ if (!BF || BF->isPLTFunction() || !BC.shouldEmit(*BF))
+ continue;
+ References.push_back({Address, Target, BF->getSymbol()});
+ }
+ }
+ const uint32_t Type = BC.isAArch64()
+ ? static_cast<uint32_t>(ELF::R_AARCH64_ABS64)
+ : static_cast<uint32_t>(ELF::R_X86_64_64);
+ for (const PointerReference &Ref : References)
+ BC.addRelocation(Ref.Address, Ref.Symbol, Type, 0, Ref.Target);
+ return Error::success();
+}
+
+} // namespace
+
+Error RelocationRecovery::runOnFunctions(BinaryContext &BC) {
+ assert(BC.RecoverRelocations && "recovery pass requires explicit opt-in");
+ if (BC.isAArch64())
+ if (Error E = recoverAArch64InstructionReferences(BC))
+ return E;
+
+ return reconstructDataRelocations(BC);
+}
+
+} // namespace bolt
+} // namespace llvm
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index e61297f94ff98..84a83ca2b6849 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -30,6 +30,7 @@
#include "bolt/Passes/PointerAuthCFIFixup.h"
#include "bolt/Passes/ProfileQualityStats.h"
#include "bolt/Passes/RegReAssign.h"
+#include "bolt/Passes/RelocationRecovery.h"
#include "bolt/Passes/ReorderData.h"
#include "bolt/Passes/ReorderFunctions.h"
#include "bolt/Passes/RetpolineInsertion.h"
@@ -85,6 +86,11 @@ cl::opt<bool>
cl::opt<bool> NeverPrint("never-print", cl::desc("never print"),
cl::ReallyHidden, cl::cat(BoltOptCategory));
+static cl::opt<bool> PrintRelocationRecovery(
+ "print-relocation-recovery",
+ cl::desc("print functions after relocation recovery"), cl::Hidden,
+ cl::cat(BoltOptCategory));
+
cl::opt<bool>
PrintAfterBranchFixup("print-after-branch-fixup",
cl::desc("print function after fixing local branches"),
@@ -374,6 +380,10 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
Manager.registerPass(
std::make_unique<PointerAuthCFIAnalyzer>(PrintPAuthCFIAnalyzer));
+ if (BC.RecoverRelocations)
+ Manager.registerPass(
+ std::make_unique<RelocationRecovery>(PrintRelocationRecovery));
+
Manager.registerPass(
std::make_unique<EstimateEdgeCounts>(PrintEstimateEdgeCounts));
diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index bc1b2ed3c2e3c..0417cc86ac0ed 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -27,6 +27,7 @@ add_llvm_library(LLVMBOLTRewrite
RSeqRewriter.cpp
SDTRewriter.cpp
GNUPropertyRewriter.cpp
+ StrippedBinary.cpp
NO_EXPORT
DISABLE_LLVM_LINK_LLVM_DYLIB
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 385b9e736fe64..c206197ccb254 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -85,6 +85,7 @@ extern cl::opt<uint32_t> InstrumentationSleepTime;
extern cl::opt<bool> KeepNops;
extern cl::opt<bool> LargeCodeModel;
extern cl::opt<bool> Lite;
+extern cl::opt<bolt::JumpTableSupportLevel> JumpTables;
extern cl::list<std::string> PrintOnly;
extern cl::opt<std::string> PrintOnlyFile;
extern cl::list<std::string> ReorderData;
@@ -102,6 +103,14 @@ static cl::opt<bool>
cl::desc("allow processing of stripped binaries"), cl::Hidden,
cl::cat(BoltCategory));
+extern cl::opt<bool> AggressiveRelocationRecovery;
+
+static cl::opt<bool> RecoverRelocations(
+ "recover-relocations",
+ cl::desc("reconstruct references for AArch64 and x86-64 ELF "
+ "inputs without static text relocations"),
+ cl::Hidden, cl::cat(BoltCategory));
+
static cl::opt<bool> ForceToDataRelocations(
"force-data-relocations",
cl::desc("force relocations to data sections to always be processed"),
@@ -824,7 +833,8 @@ Error RewriteInstance::run() {
if (Error E = readSpecialSections())
return E;
adjustCommandLineOptions();
- discoverFileObjects();
+ if (Error E = discoverFileObjects())
+ return E;
if (opts::Instrument && !BC->IsStaticExecutable) {
if (Error E = discoverRtInitAddress())
@@ -892,7 +902,7 @@ Error RewriteInstance::run() {
return Error::success();
}
-void RewriteInstance::discoverFileObjects() {
+Error RewriteInstance::discoverFileObjects() {
NamedRegionTimer T("discoverFileObjects", "discover file objects",
TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
@@ -1400,7 +1410,7 @@ void RewriteInstance::discoverFileObjects() {
continue;
}
- if (opts::Verbosity >= 1)
+ if (opts::Verbosity >= 1 && !BC->IsStripped)
BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Address)
<< ", 0x" << Twine::utohexstr(Address + FDE->getAddressRange())
<< ") has no corresponding symbol table entry\n";
@@ -1415,6 +1425,12 @@ void RewriteInstance::discoverFileObjects() {
BC->setHasSymbolsWithFileName(FileSymbols.size());
+ // Register stripped entry code and linker-generated helpers before boundary
+ // adjustment and unmarked-tail discovery process the same address ranges.
+ if (BC->RecoverRelocations && BC->IsStripped)
+ if (Error E = discoverStrippedFunctions())
+ return E;
+
// Now that all the functions were created - adjust their boundaries.
adjustFunctionBoundaries(MarkerSymbols);
splitUnmarkedTailFunctions(MarkerSymbols);
@@ -1489,6 +1505,8 @@ void RewriteInstance::discoverFileObjects() {
// The name resolver is only needed while discovering and disambiguating file
// objects. Release its memory now that all names have been uniquified.
NR.clear();
+
+ return Error::success();
}
void RewriteInstance::discoverBOLTReserved() {
@@ -2381,7 +2399,11 @@ void RewriteInstance::adjustFunctionBoundaries(
Function.setMaxSize(Function.getSize());
continue;
}
- Function.setMaxSize(MaxSize);
+ // In recovery mode, matching Size and MaxSize records an exact extent from
+ // FDE or pattern-based discovery. Preserve it instead of extending the
+ // function across unmarked code before the next discovered function.
+ if (!BC->RecoverRelocations || Function.getMaxSize() != Function.getSize())
+ Function.setMaxSize(MaxSize);
if (!Function.getSize() && Function.isSimple()) {
// Some assembly functions have their size set to 0, use the max
// size as their real size.
@@ -2527,8 +2549,39 @@ Error RewriteInstance::readSpecialSections() {
exit(1);
}
- BC->HasRelocations = HasTextRelocations &&
- (opts::RelocationMode != cl::boolOrDefault::BOU_FALSE);
+ if (opts::AggressiveRelocationRecovery && !opts::RecoverRelocations)
+ return createStringError(errc::invalid_argument,
+ "--aggressive-relocation-recovery "
+ "requires --recover-relocations");
+ BC->RecoverRelocations = opts::RecoverRelocations;
+ if (BC->RecoverRelocations) {
+ if (!BC->isAArch64() && !BC->isX86())
+ return createStringError(
+ errc::not_supported,
+ "relocation recovery requires AArch64 or x86-64");
+ if (HasTextRelocations ||
+ opts::RelocationMode != cl::boolOrDefault::BOU_UNSET)
+ return createStringError(
+ errc::invalid_argument,
+ "relocation recovery requires missing static text "
+ "relocations and cannot be combined with --relocs");
+ if (BC->IsLinuxKernel || opts::StrictMode || opts::AggregateOnly ||
+ opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
+ return createStringError(
+ errc::not_supported,
+ "relocation recovery is unavailable in this mode");
+ BC->outs()
+ << "BOLT-INFO: recovering relocations for discovered functions and "
+ "supported address references\n";
+ }
+
+ // Set HasRelocations so the relocation-mode emitter applies recovered
+ // references when functions move. RecoverRelocations separately tells code
+ // reading the input file that these references were reconstructed rather
+ // than obtained from its static relocation records.
+ BC->HasRelocations = BC->RecoverRelocations ||
+ (HasTextRelocations &&
+ opts::RelocationMode != cl::boolOrDefault::BOU_FALSE);
if (BC->IsLinuxKernel && BC->HasRelocations) {
BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";
@@ -2576,6 +2629,13 @@ Error RewriteInstance::readSpecialSections() {
}
void RewriteInstance::adjustCommandLineOptions() {
+ if (BC->RecoverRelocations) {
+ if (opts::JumpTables != JTS_MOVE)
+ BC->outs() << "BOLT-INFO: forcing --jump-tables=move for relocation "
+ "recovery\n";
+ opts::JumpTables = JTS_MOVE;
+ }
+
if (BC->isAArch64() && !BC->HasRelocations)
BC->errs() << "BOLT-WARNING: non-relocation mode for AArch64 is not fully "
"supported\n";
@@ -6129,7 +6189,10 @@ void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
}
}
if (!SymTabSection) {
- BC->errs() << "BOLT-WARNING: no symbol table found\n";
+ if (BC->RecoverRelocations)
+ BC->outs() << "BOLT-INFO: input has no symbol table\n";
+ else
+ BC->errs() << "BOLT-WARNING: no symbol table found\n";
return;
}
diff --git a/bolt/lib/Rewrite/StrippedBinary.cpp b/bolt/lib/Rewrite/StrippedBinary.cpp
new file mode 100644
index 0000000000000..0b60deabb078f
--- /dev/null
+++ b/bolt/lib/Rewrite/StrippedBinary.cpp
@@ -0,0 +1,285 @@
+//===- bolt/Rewrite/StrippedBinary.cpp - Stripped ELF helpers -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Rewrite/RewriteInstance.h"
+#include "llvm/MC/MCDisassembler/MCDisassembler.h"
+#include "llvm/Object/ELFObjectFile.h"
+
+#include <algorithm>
+#include <optional>
+
+namespace llvm {
+namespace bolt {
+namespace {
+
+/// Decode the 4-byte instruction at Address. Reject unaligned addresses and
+/// addresses outside the file-backed contents of an executable section.
+bool decodeAt(BinaryContext &BC, uint64_t Address, MCInst &Inst) {
+ auto Section = BC.getSectionForAddress(Address);
+ if (!Section || !Section->isText() || Address % 4)
+ return false;
+ const StringRef Contents = Section->getContents();
+ const uint64_t Offset = Address - Section->getAddress();
+ if (Offset > Contents.size() || Contents.size() - Offset < 4)
+ return false;
+ const auto *Bytes = reinterpret_cast<const uint8_t *>(Contents.data());
+ uint64_t Size = 0;
+ return BC.DisAsm->getInstruction(
+ Inst, Size, ArrayRef<uint8_t>(Bytes + Offset, 4), Address,
+ nulls()) == MCDisassembler::Success &&
+ Size == 4;
+}
+
+/// Return true for a direct unconditional branch, excluding indirect
+/// branches and returns.
+bool isDirectBranch(BinaryContext &BC, const MCInst &Inst) {
+ return BC.MIB->isUnconditionalBranch(Inst) &&
+ !BC.MIB->isIndirectBranch(Inst) && !BC.MIB->isReturn(Inst);
+}
+
+/// Return true when Inst can prefix glibc's optional main wrapper. The prefix
+/// is NOP without branch target identification and BTI C when it is enabled.
+/// LLVM represents them as HINT instructions with immediates 0 and 34.
+bool isAArch64WrapperPrefix(BinaryContext &BC, const MCInst &Inst) {
+ if (BC.MIB->isNoop(Inst))
+ return true;
+ return BC.MII->getName(Inst.getOpcode()) == "HINT" &&
+ Inst.getNumOperands() != 0 && Inst.getOperand(0).isImm() &&
+ Inst.getOperand(0).getImm() == 34;
+}
+
+/// Match a direct, non-tail call in stripped AArch64 entry code. For an input
+/// with a dynamic section, require the target name Name or Name at PLT. For a
+/// static input, accept any registered direct target; the caller verifies its
+/// position in the ordered glibc startup sequence.
+bool matchesAArch64EntryCall(BinaryContext &BC, const MCInst &Inst,
+ uint64_t Address, StringRef Name,
+ bool HasDynamicSection) {
+ uint64_t Target = 0;
+ if (!BC.MIB->isCall(Inst) || BC.MIB->isTailCall(Inst) ||
+ !BC.MIB->evaluateBranch(Inst, Address, 4, Target))
+ return false;
+ const BinaryFunction *BF = BC.getBinaryFunctionAtAddress(Target);
+ if (!BF)
+ return false;
+ return !HasDynamicSection || BF->hasName(Name.str()) ||
+ BF->hasName(Name.str() + "@PLT");
+}
+
+/// Extent of a recognized _start sequence and the optional offset at which
+/// its adjacent main wrapper can also be entered.
+struct AArch64EntryMatch {
+ uint64_t Size;
+ std::optional<uint64_t> SecondaryEntryOffset;
+};
+
+/// Scan [Entry, End) for the AArch64 glibc startup sequence. It consists of a
+/// call to __libc_start_main immediately followed by a call to abort. After
+/// those calls, NOP or BTI C prefixes followed by a direct branch to a known
+/// function identify an adjacent main wrapper. Report the first prefix, or
+/// the branch itself when there is no prefix, as a secondary entry to _start.
+std::optional<AArch64EntryMatch>
+matchAArch64EntryPattern(BinaryContext &BC, uint64_t Entry, uint64_t End,
+ bool HasDynamicSection) {
+ enum class MatchState { StartMain, Abort, Wrapper };
+ MatchState State = MatchState::StartMain;
+ uint64_t Size = 0;
+ std::optional<uint64_t> WrapperPrefixAddress;
+
+ for (uint64_t Address = Entry; Address < End && End - Address >= 4;
+ Address += 4) {
+ MCInst Inst;
+ if (!decodeAt(BC, Address, Inst))
+ break;
+
+ if (State == MatchState::StartMain) {
+ if (matchesAArch64EntryCall(BC, Inst, Address, "__libc_start_main",
+ HasDynamicSection)) {
+ State = MatchState::Abort;
+ continue;
+ }
+ if (BC.MIB->isCall(Inst) || BC.MIB->isBranch(Inst) ||
+ BC.MIB->isReturn(Inst))
+ break;
+ continue;
+ }
+
+ if (State == MatchState::Abort) {
+ if (!matchesAArch64EntryCall(BC, Inst, Address, "abort",
+ HasDynamicSection))
+ break;
+ Size = Address + 4 - Entry;
+ State = MatchState::Wrapper;
+ continue;
+ }
+
+ if (isAArch64WrapperPrefix(BC, Inst)) {
+ if (!WrapperPrefixAddress)
+ WrapperPrefixAddress = Address;
+ continue;
+ }
+ if (!isDirectBranch(BC, Inst))
+ return AArch64EntryMatch{Size, std::nullopt};
+
+ uint64_t Target = 0;
+ if (!BC.MIB->evaluateBranch(Inst, Address, 4, Target) ||
+ !BC.getBinaryFunctionAtAddress(Target))
+ return AArch64EntryMatch{Size, std::nullopt};
+ const uint64_t WrapperAddress = WrapperPrefixAddress.value_or(Address);
+ return AArch64EntryMatch{Address + 4 - Entry, WrapperAddress - Entry};
+ }
+
+ if (State == MatchState::Wrapper)
+ return AArch64EntryMatch{Size, std::nullopt};
+ return std::nullopt;
+}
+
+/// Recover AArch64 _start from the ELF entry address. Limit the scan to the
+/// existing entry function, the next known function, or file-backed executable
+/// contents. If _start is missing, create it only after matching the glibc
+/// startup sequence. If it already exists, keep its bounds and use a match only
+/// to add the main-wrapper secondary entry.
+Error discoverAArch64Entry(BinaryContext &BC, uint64_t Entry) {
+ assert(BC.isAArch64() && "AArch64 entry discovery requires AArch64 input");
+ if (!Entry)
+ return Error::success(); // Shared objects commonly have no process entry.
+ BinaryFunction *Start = BC.getBinaryFunctionAtAddress(Entry);
+ auto Section = BC.getSectionForAddress(Entry);
+ if (!Section || !Section->isText() || Entry % 4)
+ return createFatalBOLTError("invalid stripped AArch64 ELF entry point");
+ if (!Start && BC.getBinaryFunctionContainingAddress(Entry))
+ return createFatalBOLTError("ELF entry overlaps a discovered function");
+
+ uint64_t End = Section->getAddress() + Section->getContents().size();
+ if (Entry >= End)
+ return createFatalBOLTError(
+ "stripped AArch64 ELF entry has no file-backed instruction");
+ if (Start) {
+ // FDE discovery already supplied the entry function's exact boundary.
+ End = std::min(End, Entry + Start->getSize());
+ } else {
+ // The next FDE-backed function bounds an unregistered _start candidate. If
+ // none exists, cap the scan within the file-backed executable contents.
+ auto Next = BC.getBinaryFunctions().upper_bound(Entry);
+ if (Next != BC.getBinaryFunctions().end())
+ End = std::min(End, Next->first);
+ else
+ End = Entry + std::min<uint64_t>(End - Entry, 4096);
+ }
+
+ const bool HasDynamicSection = bool(BC.getUniqueSectionByName(".dynamic"));
+ const std::optional<AArch64EntryMatch> Match =
+ matchAArch64EntryPattern(BC, Entry, End, HasDynamicSection);
+ if (Match) {
+ // Register a new function only after the bounded pattern matches. An
+ // existing FDE-backed _start keeps its boundary and gains only the
+ // secondary entry identified by the pattern.
+ if (!Start) {
+ Start = BC.createBinaryFunction("_start", *Section, Entry, Match->Size);
+ Start->setMaxSize(Match->Size);
+ BC.outs() << "BOLT-INFO: recovered stripped entry at 0x"
+ << Twine::utohexstr(Entry) << '\n';
+ }
+ if (Match->SecondaryEntryOffset)
+ Start->addEntryPointAtOffset(*Match->SecondaryEntryOffset);
+ return Error::success();
+ }
+ // An existing FDE-backed entry already has a function boundary. An
+ // unregistered entry must match the startup pattern before it is added.
+ if (Start)
+ return Error::success();
+ return createFatalBOLTError(
+ "cannot recover stripped entry: expected a bounded "
+ "glibc __libc_start_main/abort sequence");
+}
+
+/// Find linker-generated Cortex-A53 erratum 843419 veneers reached from known
+/// functions. The source ADRP must occupy one of the last two instruction slots
+/// in a 4 KiB page: its address has low 12 bits 0xFF8 or 0xFFC. From that ADRP,
+/// look for a nearby direct branch to an unregistered two-instruction helper
+/// that performs a load or store using the ADRP destination register and then
+/// branches back to the instruction after the original branch. Register each
+/// helper before normal function disassembly, which must not modify the
+/// function map.
+///
+/// See the Arm Cortex-A53 MPCore Software Developers Errata Notice, erratum
+/// 843419.
+void discoverErratumVeneers(BinaryContext &BC) {
+ SmallVector<BinaryFunction *, 16> Functions;
+ for (auto &BFI : BC.getBinaryFunctions())
+ Functions.push_back(&BFI.second);
+ for (BinaryFunction *BF : Functions) {
+ for (uint64_t Offset = 0; Offset < BF->getSize(); Offset += 4) {
+ const uint64_t Address = BF->getAddress() + Offset;
+ if ((Address & 4095) != 4088 && (Address & 4095) != 4092)
+ continue;
+ MCInst ADRP;
+ if (!decodeAt(BC, Address, ADRP) || !BC.MIB->isADRP(ADRP))
+ continue;
+ const unsigned Reg = ADRP.getOperand(0).getReg();
+ for (unsigned Distance = 4;
+ Distance <= 16 && Offset + Distance < BF->getSize(); Distance += 4) {
+ const uint64_t BranchAddress = Address + Distance;
+ MCInst Branch;
+ if (!decodeAt(BC, BranchAddress, Branch))
+ break;
+ if (!isDirectBranch(BC, Branch)) {
+ if (BC.MIB->isBranch(Branch) || BC.MIB->isCall(Branch) ||
+ BC.MIB->isReturn(Branch) ||
+ (BC.MIB->hasDefOfPhysReg(Branch, Reg) &&
+ !BC.MIB->hasUseOfPhysReg(Branch, Reg)))
+ break;
+ continue;
+ }
+ uint64_t Target = 0, ReturnAddress = 0;
+ if (!BC.MIB->evaluateBranch(Branch, BranchAddress, 4, Target) ||
+ BC.getBinaryFunctionContainingAddress(Target) ||
+ BC.getBinaryFunctionContainingAddress(Target + 7))
+ break;
+ auto Section = BC.getSectionForAddress(Target);
+ MCInst Memory, Return;
+ if (!Section || !decodeAt(BC, Target, Memory) ||
+ !decodeAt(BC, Target + 4, Return) ||
+ (!BC.MIB->mayLoad(Memory) && !BC.MIB->mayStore(Memory)) ||
+ !BC.MIB->hasUseOfPhysReg(Memory, Reg) ||
+ !isDirectBranch(BC, Return) ||
+ !BC.MIB->evaluateBranch(Return, Target + 4, 4, ReturnAddress) ||
+ ReturnAddress != BranchAddress + 4 ||
+ ReturnAddress >= BF->getAddress() + BF->getSize())
+ break;
+ BinaryFunction *Veneer = BC.createBinaryFunction(
+ "__BOLT_e843419_" + Twine::utohexstr(Target).str(), *Section,
+ Target, 8);
+ Veneer->setMaxSize(8);
+ break;
+ }
+ }
+ }
+}
+
+} // namespace
+
+/// Recover functions that FDE discovery can miss in a stripped ELF file.
+/// Currently this discovers AArch64 _start and erratum 843419 veneers.
+Error RewriteInstance::discoverStrippedFunctions() {
+ const auto *ELF = dyn_cast<object::ELF64LEObjectFile>(InputFile);
+ if (!ELF)
+ return createFatalBOLTError(
+ "stripped recovery requires little-endian ELF64");
+ if (BC->isAArch64()) {
+ if (Error E =
+ discoverAArch64Entry(*BC, ELF->getELFFile().getHeader().e_entry))
+ return E;
+ discoverErratumVeneers(*BC);
+ }
+ return Error::success();
+}
+
+} // namespace bolt
+} // namespace llvm
diff --git a/bolt/test/AArch64/recover-relocations-ambiguous.s b/bolt/test/AArch64/recover-relocations-ambiguous.s
new file mode 100644
index 0000000000000..78eedeb29474e
--- /dev/null
+++ b/bolt/test/AArch64/recover-relocations-ambiguous.s
@@ -0,0 +1,33 @@
+## A page can feed addresses with different low bits. Preserve the absolute
+## input page in the ADRP and leave both ADD instructions unchanged.
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-linux %s -o %t.o
+# RUN: ld.lld --no-relax %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --recover-relocations \
+# RUN: --print-relocation-recovery --print-only=_start 2>&1 | FileCheck %s
+# CHECK: after recover-relocations
+# CHECK: adrp x0, {{.*}}__BOLT_zero_addr
+# CHECK-NEXT: add x1, x0, #0x{{[0-9a-f]+}}
+# CHECK-NEXT: add x2, x0, #0x{{[0-9a-f]+}}
+.text
+.globl _start
+.type _start, %function
+_start:
+.cfi_startproc
+ adrp x0, first
+ add x1, x0, :lo12:first
+ add x2, x0, :lo12:second
+ ret
+.cfi_endproc
+.size _start, .-_start
+.type first, %function
+first:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size first, .-first
+.type second, %function
+second:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size second, .-second
diff --git a/bolt/test/AArch64/recover-relocations.s b/bolt/test/AArch64/recover-relocations.s
new file mode 100644
index 0000000000000..1edd8abe699fd
--- /dev/null
+++ b/bolt/test/AArch64/recover-relocations.s
@@ -0,0 +1,92 @@
+## Recover a function address after code movement and preserve the original
+## page address used to access a data object.
+## No --emit-relocs: these operands must be reconstructed from instructions.
+# RUN: echo target > %t.order
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-linux %s -o %t.o
+# RUN: ld.lld --no-relax %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --recover-relocations \
+# RUN: --print-relocation-recovery --print-only=_start \
+# RUN: --reorder-functions=user --function-order=%t.order \
+# RUN: 2>&1 | FileCheck %s
+# RUN: %python %p/../Inputs/check-recovered-addresses.py aarch64 %t.exe %t.bolt
+# RUN: llvm-readelf -h %t.bolt | FileCheck %s --check-prefix=ELF
+# RUN: not llvm-bolt %t.exe -o %t.bad --recover-relocations \
+# RUN: --relocs=0 2>&1 | FileCheck %s --check-prefix=MODE
+# RUN: ld.lld --emit-relocs --no-relax %t.o -o %t.reloc
+# RUN: not llvm-bolt %t.reloc -o %t.bad --recover-relocations \
+# RUN: 2>&1 | FileCheck %s --check-prefix=MODE
+# RUN: llvm-bolt %t.reloc -o %t.normal --reorder-functions=user --function-order=%t.order
+
+# CHECK: BOLT-INFO: forcing --jump-tables=move for relocation recovery
+# CHECK: after recover-relocations
+# CHECK: adrp x0, target
+# CHECK: add x0, x0, :lo12:target
+# CHECK: adrp x1, {{.*}}__BOLT_zero_addr
+# CHECK-COUNT-2: adrp x3, target
+# CHECK: add x3, x3, :lo12:target
+# CHECK: adrp x4, {{.*}}__BOLT_zero_addr
+# CHECK: add x4, x4, #{{(0x)?[0-9a-f]+}}
+# CHECK-COUNT-2: adrp x5, {{.*}}__BOLT_zero_addr
+# CHECK: add x5, x5, #{{(0x)?[0-9a-f]+}}
+# MODE: relocation recovery requires missing static text relocations
+# ELF: Entry point address: 0x{{[1-9a-fA-F][0-9a-fA-F]*}}
+
+.text
+.globl _start
+.type _start, %function
+_start:
+.cfi_startproc
+ adrp x0, target
+ add x0, x0, :lo12:target
+ adrp x1, object
+ ldr x2, [x1, :lo12:object]
+ cbz x2, .Lleft
+ adrp x3, target
+ b .Ljoin
+.Lleft:
+ adrp x3, target
+.Ljoin:
+ add x3, x3, :lo12:target
+
+ // A non-ADRP definition on one incoming path makes the join unsafe.
+ cbz x2, .Lunknown
+ adrp x4, target
+ b .Lunknown_join
+.Lunknown:
+ mov x4, xzr
+.Lunknown_join:
+ add x4, x4, :lo12:target
+
+ // ADRPs with different pages at a join must both fall back.
+ cbz x2, .Lother_page
+ adrp x5, target
+ b .Lpage_join
+.Lother_page:
+ adrp x5, other
+.Lpage_join:
+ add x5, x5, :lo12:target
+ ret
+.cfi_endproc
+.size _start, .-_start
+
+.p2align 12
+.globl target
+.type target, %function
+target:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size target, .-target
+
+.p2align 12
+.type other, %function
+other:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size other, .-other
+
+.data
+.p2align 12
+object:
+ .xword 42
diff --git a/bolt/test/AArch64/stripped-entry.s b/bolt/test/AArch64/stripped-entry.s
new file mode 100644
index 0000000000000..aa075af6c74be
--- /dev/null
+++ b/bolt/test/AArch64/stripped-entry.s
@@ -0,0 +1,52 @@
+## Recover an AArch64 glibc-style entry which has no FDE or symbol-table entry.
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-linux %s -o %t.o
+# RUN: ld.lld --export-dynamic %t.o -o %t.exe
+# RUN: llvm-strip --strip-all %t.exe -o %t.stripped
+# RUN: llvm-bolt %t.stripped -o %t.bolt --allow-stripped \
+# RUN: --recover-relocations --print-relocation-recovery --print-only=_start \
+# RUN: 2>&1 | FileCheck %s
+# RUN: llvm-readelf -h %t.bolt | FileCheck %s --check-prefix=ELF
+
+# CHECK: BOLT-INFO: recovered stripped entry at
+# CHECK: Binary Function "_start"
+# CHECK: Secondary Entry Points : __ENTRY__start
+# CHECK: adrp x0, __ENTRY_
+# CHECK-NEXT: add x0, x0, :lo12:__ENTRY_
+# ELF: Entry point address: 0x{{[1-9a-fA-F][0-9a-fA-F]*}}
+
+.text
+.globl _start
+.hidden _start
+.type _start, %function
+_start:
+ adrp x0, .Lmain_wrapper
+ add x0, x0, :lo12:.Lmain_wrapper
+ bl __libc_start_main
+ bl abort
+.Lmain_wrapper:
+ b main
+.size _start, .-_start
+
+.globl __libc_start_main
+.type __libc_start_main, %function
+__libc_start_main:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size __libc_start_main, .-__libc_start_main
+
+.globl abort
+.type abort, %function
+abort:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size abort, .-abort
+
+.globl main
+.type main, %function
+main:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size main, .-main
diff --git a/bolt/test/AArch64/stripped-erratum-843419.s b/bolt/test/AArch64/stripped-erratum-843419.s
new file mode 100644
index 0000000000000..f3250573ffe71
--- /dev/null
+++ b/bolt/test/AArch64/stripped-erratum-843419.s
@@ -0,0 +1,57 @@
+## Recover an unnamed Cortex-A53 erratum 843419 veneer. The ADRP sits at
+## page offset 0xffc. An intervening load follows the address dependency while
+## redefining the register, and the veneer uses the resulting value before
+## returning to the instruction after the redirected branch.
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-linux %t/input.s -o %t.o
+# RUN: ld.lld -T %t/layout.ld %t.o -o %t.exe
+# RUN: llvm-strip --strip-all %t.exe -o %t.stripped
+# RUN: llvm-bolt %t.stripped -o %t.bolt --allow-stripped \
+# RUN: --recover-relocations --print-disasm \
+# RUN: --print-only=__BOLT_e843419_212000 2>&1 | FileCheck %s
+
+# CHECK: Binary Function "__BOLT_e843419_212000"
+# CHECK: ldr x1, [x0]
+# CHECK-NEXT: b
+
+#--- input.s
+.section .text.source,"ax",%progbits
+.globl source
+.hidden source
+.type source, %function
+source:
+.cfi_startproc
+ .space 0xffc
+ adrp x0, pointer
+ ldr x0, [x0, :lo12:pointer]
+ b veneer
+.Lreturn:
+ ret
+.cfi_endproc
+.size source, .-source
+
+.section .text.veneer,"ax",%progbits
+.type veneer, %function
+veneer:
+ ldr x1, [x0, :lo12:object]
+ b .Lreturn
+.size veneer, .-veneer
+
+.data
+.p2align 3
+object:
+ .xword 42
+pointer:
+ .xword object
+
+#--- layout.ld
+ENTRY(source)
+SECTIONS {
+ . = 0x210000;
+ .text : { *(.text.source) }
+ . = 0x212000;
+ .text.veneer : { *(.text.veneer) }
+ . = 0x220000;
+ .data : { *(.data) }
+ .eh_frame : { *(.eh_frame) }
+}
diff --git a/bolt/test/Inputs/check-recovered-addresses.py b/bolt/test/Inputs/check-recovered-addresses.py
new file mode 100644
index 0000000000000..e622453e7c195
--- /dev/null
+++ b/bolt/test/Inputs/check-recovered-addresses.py
@@ -0,0 +1,75 @@
+# Validate recovered addresses by reading the input and output ELF files rather
+# than matching llvm-bolt diagnostics. All modes verify that target moved and
+# that the ELF entry points to the recovered _start. The AArch64 mode decodes
+# the ADRP/ADD address in _start. The x86 modes check a data pointer and the
+# .init_array entry, including the pointer left unchanged in conservative mode.
+# Usage: check-recovered-addresses.py {aarch64,x86,x86-conservative} input output
+import struct
+import sys
+
+
+class ELF:
+ def __init__(self, path):
+ self.data = open(path, "rb").read()
+ assert self.data[:6] == b"\x7fELF\x02\x01", "expected ELF64LE"
+ header = struct.unpack_from("<HHIQQQIHHHHHH", self.data, 16)
+ self.entry = header[3]
+ shoff, shsize, shnum, shstr = header[5], header[10], header[11], header[12]
+ self.sections = [
+ struct.unpack_from("<IIQQQQIIQQ", self.data, shoff + i * shsize)
+ for i in range(shnum)
+ ]
+ names = self.contents(self.sections[shstr])
+ self.named_sections = {self.string(names, s[0]): s for s in self.sections}
+ self.symbols = {}
+ for section in self.sections:
+ if section[1] != 2: # SHT_SYMTAB
+ continue
+ strings = self.contents(self.sections[section[6]])
+ for offset in range(section[4], section[4] + section[5], section[9]):
+ name, info, other, index, value, size = struct.unpack_from(
+ "<IBBHQQ", self.data, offset
+ )
+ self.symbols[self.string(strings, name)] = value
+
+ @staticmethod
+ def string(data, offset):
+ return data[offset : data.index(b"\0", offset)].decode()
+
+ def contents(self, section):
+ return self.data[section[4] : section[4] + section[5]]
+
+ def at(self, address, size):
+ for section in self.sections:
+ if section[2] & 2 and section[1] != 8:
+ start = address - section[3]
+ if 0 <= start and start + size <= section[5]:
+ return self.contents(section)[start : start + size]
+ raise AssertionError(f"unmapped address {address:#x}")
+
+
+arch, before, after = sys.argv[1:]
+old, new = ELF(before), ELF(after)
+target = new.symbols["target"]
+assert target != old.symbols["target"], "test must move target"
+assert new.entry == new.symbols["_start"], "ELF entry must follow recovered start"
+if arch == "aarch64":
+ start = new.symbols["_start"]
+ adrp, add = struct.unpack("<II", new.at(start, 8))
+ assert adrp & 0x9F000000 == 0x90000000, "expected ADRP"
+ assert add & 0xFF000000 == 0x91000000, "expected ADD immediate"
+ immediate = ((adrp >> 29) & 3) | (((adrp >> 5) & 0x7FFFF) << 2)
+ if immediate & (1 << 20):
+ immediate -= 1 << 21
+ address = (start & ~4095) + (immediate << 12) + ((add >> 10) & 4095)
+ assert address == target, (hex(address), hex(target))
+elif arch in ("x86", "x86-conservative"):
+ pointer = struct.unpack("<Q", new.at(new.symbols["pointer"], 8))[0]
+ array = struct.unpack("<Q", new.contents(new.named_sections[".init_array"])[:8])[0]
+ if arch == "x86":
+ assert pointer == target, (hex(pointer), hex(target))
+ else:
+ assert pointer == old.symbols["target"], (hex(pointer), hex(old.symbols["target"]))
+ assert array == target, (hex(array), hex(target))
+else:
+ raise AssertionError(f"unexpected architecture: {arch}")
diff --git a/bolt/test/X86/recover-relocations-data.s b/bolt/test/X86/recover-relocations-data.s
new file mode 100644
index 0000000000000..4581d45151b78
--- /dev/null
+++ b/bolt/test/X86/recover-relocations-data.s
@@ -0,0 +1,45 @@
+## Conservative mode updates pointer arrays only. Aggressive mode also treats
+## matching aligned words in selected general data sections as pointers.
+# RUN: echo target > %t.order
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.conservative --recover-relocations \
+# RUN: --reorder-functions=user --function-order=%t.order
+# RUN: %python %p/../Inputs/check-recovered-addresses.py x86-conservative \
+# RUN: %t.exe %t.conservative
+# RUN: not llvm-bolt %t.exe -o %t.bad \
+# RUN: --aggressive-relocation-recovery \
+# RUN: 2>&1 | FileCheck %s --check-prefix=OPTION
+# RUN: llvm-bolt %t.exe -o %t.bolt --recover-relocations \
+# RUN: --aggressive-relocation-recovery \
+# RUN: --reorder-functions=user --function-order=%t.order
+# RUN: %python %p/../Inputs/check-recovered-addresses.py x86 %t.exe %t.bolt
+# RUN: llvm-nm %t.bolt | FileCheck %s --check-prefix=SYMBOL
+# OPTION: --aggressive-relocation-recovery requires --recover-relocations
+# SYMBOL: T target
+
+.text
+.globl _start
+.type _start, @function
+_start:
+.cfi_startproc
+ call target
+ ret
+.cfi_endproc
+.size _start, .-_start
+.globl target
+.type target, @function
+target:
+.cfi_startproc
+ ret
+.cfi_endproc
+.size target, .-target
+.data
+.p2align 3
+pointer:
+ .quad target
+## Short tail must not be read as a complete pointer.
+ .byte 1, 2, 3
+.section .init_array,"aw", at init_array
+.p2align 3
+ .quad target
diff --git a/bolt/test/X86/stripped-jump-table.s b/bolt/test/X86/stripped-jump-table.s
new file mode 100644
index 0000000000000..3d8c5bfdfe6d2
--- /dev/null
+++ b/bolt/test/X86/stripped-jump-table.s
@@ -0,0 +1,49 @@
+## A stripped input has no symbol evidence for fragment relationships. Accept
+## entries that resolve to valid instructions in discovered functions and force
+## movable jump tables.
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+# RUN: ld.lld -e _start %t.o -o %t.exe
+# RUN: llvm-strip --strip-all %t.exe -o %t.stripped
+# RUN: llvm-bolt %t.stripped -o %t.bolt --allow-stripped \
+# RUN: --recover-relocations --print-normalized 2>&1 | FileCheck %s
+
+# CHECK: BOLT-INFO: forcing --jump-tables=move for relocation recovery
+# CHECK: jmpq {{.*}} # JUMPTABLE
+
+.text
+.globl _start
+.type _start, @function
+_start:
+.cfi_startproc
+ callq dispatch
+ retq
+.cfi_endproc
+.size _start, .-_start
+
+.globl dispatch
+.type dispatch, @function
+dispatch:
+.cfi_startproc
+ andl $1, %edi
+ leaq .LJTI0(%rip), %rax
+ movslq (%rax,%rdi,4), %rcx
+ addq %rax, %rcx
+ jmpq *%rcx
+.Llocal:
+ retq
+.cfi_endproc
+.size dispatch, .-dispatch
+
+.globl other
+.type other, @function
+other:
+.cfi_startproc
+ retq
+.cfi_endproc
+.size other, .-other
+
+.section .rodata,"a", at progbits
+.p2align 2
+.LJTI0:
+ .long .Llocal-.LJTI0
+ .long other-.LJTI0
diff --git a/llvm/utils/gn/secondary/bolt/lib/Passes/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Passes/BUILD.gn
index 3b519c0a4cfc0..a45e1dc056cd6 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Passes/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Passes/BUILD.gn
@@ -24,6 +24,7 @@ static_library("Passes") {
"DataflowInfoManager.cpp",
"FixRISCVCallsPass.cpp",
"FixRelaxationPass.cpp",
+ "RelocationRecovery.cpp",
"FrameAnalysis.cpp",
"FrameOptimizer.cpp",
"HFSort.cpp",
diff --git a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
index 4ab1c95b430a8..646131fe4b0f8 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
@@ -37,6 +37,7 @@ static_library("Rewrite") {
"RSeqRewriter.cpp",
"RewriteInstance.cpp",
"SDTRewriter.cpp",
+ "StrippedBinary.cpp",
]
defines = []
More information about the llvm-commits
mailing list