[llvm] [BOLT][Linux] Add support for relocation mode (PR #130931)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 12 02:17:09 PDT 2025
https://github.com/FLZ101 created https://github.com/llvm/llvm-project/pull/130931
None
>From 49dc8ea7b820db7d5473bf4af7a2f8085fa47c36 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Sat, 8 Mar 2025 14:05:24 +0800
Subject: [PATCH 1/6] [BOLT][Linux] Refactor to support different architectures
and Linux versions
* utilize LinuxKernelVersion to simplify parsing of alternative
instruction entries and exception table entries
* refactor static keys handling to make it work with relocation
mode and function splitting, and also reduce duplicated code
---
bolt/docs/CommandLineArgumentReference.md | 12 -
bolt/include/bolt/Core/BinaryContext.h | 13 +-
bolt/include/bolt/Core/BinaryFunction.h | 11 +
bolt/include/bolt/Core/FunctionLayout.h | 4 +
bolt/include/bolt/Core/Linker.h | 7 +
bolt/include/bolt/Core/MCPlusBuilder.h | 3 +-
bolt/include/bolt/Rewrite/MetadataRewriter.h | 8 +-
bolt/include/bolt/Rewrite/MetadataRewriters.h | 8 +-
bolt/include/bolt/Rewrite/RewriteInstance.h | 2 +
bolt/lib/Core/BinaryContext.cpp | 17 +
bolt/lib/Core/JumpTable.cpp | 2 +-
bolt/lib/Core/MCPlusBuilder.cpp | 9 +-
bolt/lib/Rewrite/BuildIDRewriter.cpp | 8 +-
bolt/lib/Rewrite/CMakeLists.txt | 1 +
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 411 +++++++++---------
bolt/lib/Rewrite/MetadataRewriter.cpp | 20 +
bolt/lib/Rewrite/PseudoProbeRewriter.cpp | 8 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 19 +-
bolt/lib/Rewrite/SDTRewriter.cpp | 7 +-
bolt/test/X86/linux-alt-instruction.s | 38 +-
20 files changed, 324 insertions(+), 284 deletions(-)
create mode 100644 bolt/lib/Rewrite/MetadataRewriter.cpp
diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index f3881c9a640a9..2c880388e3faa 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -56,14 +56,6 @@
Allow processing of stripped binaries
-- `--alt-inst-feature-size=<uint>`
-
- Size of feature field in .altinstructions
-
-- `--alt-inst-has-padlen`
-
- Specify that .altinstructions has padlen field
-
- `--asm-dump[=<dump folder>]`
Dump function into assembly
@@ -254,10 +246,6 @@
Redirect journaling to a file instead of stdout/stderr
-- `--long-jump-labels`
-
- Always use long jumps/nops for Linux kernel static keys
-
- `--match-profile-with-function-hash`
Match profile with function hash
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 8bec1db70e25a..77104d4c2f9cc 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -911,7 +911,11 @@ class BinaryContext {
/// Return a value of the global \p Symbol or an error if the value
/// was not set.
ErrorOr<uint64_t> getSymbolValue(const MCSymbol &Symbol) const {
- const BinaryData *BD = getBinaryDataByName(Symbol.getName());
+ return getSymbolValue(Symbol.getName());
+ }
+
+ ErrorOr<uint64_t> getSymbolValue(StringRef Name) const {
+ const BinaryData *BD = getBinaryDataByName(Name);
if (!BD)
return std::make_error_code(std::errc::bad_address);
return BD->getAddress();
@@ -1237,6 +1241,13 @@ class BinaryContext {
return const_cast<BinaryContext *>(this)->getSectionForAddress(Address);
}
+ ErrorOr<BinarySection &> getSectionForOutputAddress(uint64_t Address);
+ ErrorOr<const BinarySection &>
+ getSectionForOutputAddress(uint64_t Address) const {
+ return const_cast<BinaryContext *>(this)->getSectionForOutputAddress(
+ Address);
+ }
+
/// Return internal section representation for a section in a file.
BinarySection *getSectionForSectionRef(SectionRef Section) const {
return SectionRefToBinarySection.lookup(Section);
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 942840a7621fd..20caebe4b129c 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1243,6 +1243,17 @@ class BinaryFunction {
return Islands->FunctionColdConstantIslandLabel;
}
+ const FunctionFragment *
+ getFunctionFragmentForOutputAddress(uint64_t OutputAddress) const {
+ for (const FunctionFragment &FF : Layout.fragments()) {
+ uint64_t Address = FF.getAddress();
+ uint64_t Size = FF.getImageSize();
+ if (Address <= OutputAddress && OutputAddress < Address + Size)
+ return &FF;
+ }
+ return nullptr;
+ }
+
/// Return true if this is a function representing a PLT entry.
bool isPLTFunction() const { return PLTSymbol != nullptr; }
diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h
index ee4dd689b8dd6..65b80051862c4 100644
--- a/bolt/include/bolt/Core/FunctionLayout.h
+++ b/bolt/include/bolt/Core/FunctionLayout.h
@@ -117,6 +117,10 @@ class FunctionFragment {
uint64_t getFileOffset() const { return FileOffset; }
void setFileOffset(uint64_t Offset) { FileOffset = Offset; }
+ uint8_t *getOutputData() const {
+ return reinterpret_cast<uint8_t *>(getImageAddress());
+ }
+
unsigned size() const { return Size; };
bool empty() const { return size() == 0; };
iterator begin();
diff --git a/bolt/include/bolt/Core/Linker.h b/bolt/include/bolt/Core/Linker.h
index 66b3ad18e3c7b..1e0876a0e13d9 100644
--- a/bolt/include/bolt/Core/Linker.h
+++ b/bolt/include/bolt/Core/Linker.h
@@ -46,6 +46,13 @@ class BOLTLinker {
/// Return the address and size of a symbol or std::nullopt if it cannot be
/// found.
virtual std::optional<SymbolInfo> lookupSymbolInfo(StringRef Name) const = 0;
+
+ /// Return the address of a symbol or std::nullopt if it cannot be found.
+ std::optional<uint64_t> lookupSymbol(StringRef Name) const {
+ if (const auto Info = lookupSymbolInfo(Name))
+ return Info->Address;
+ return std::nullopt;
+ }
};
} // namespace bolt
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index fbb853656fb91..52643ffcd5b78 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1211,8 +1211,9 @@ class MCPlusBuilder {
/// Set the label of \p Inst or return the existing label for the instruction.
/// This label will be emitted right before \p Inst is emitted to MCStreamer.
+ /// If \p Temp is true, then this label does not survive in the symbol table.
MCSymbol *getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
- MCContext *Ctx) const;
+ MCContext *Ctx, bool Temp = true) const;
/// Set the label of \p Inst. This label will be emitted right before \p Inst
/// is emitted to MCStreamer.
diff --git a/bolt/include/bolt/Rewrite/MetadataRewriter.h b/bolt/include/bolt/Rewrite/MetadataRewriter.h
index 6ff8f0af7a8e6..6988e5de4e6bd 100644
--- a/bolt/include/bolt/Rewrite/MetadataRewriter.h
+++ b/bolt/include/bolt/Rewrite/MetadataRewriter.h
@@ -19,6 +19,8 @@
namespace llvm {
namespace bolt {
+class RewriteInstance;
+
/// Base class for handling file sections with metadata. In this context,
/// metadata encompasses a wide range of data that references code and other
/// data. Such metadata may or may not have an impact on program execution.
@@ -34,10 +36,14 @@ class MetadataRewriter {
StringRef Name;
protected:
+ RewriteInstance &RI;
+
/// Provides access to the binary context.
BinaryContext &BC;
- MetadataRewriter(StringRef Name, BinaryContext &BC) : Name(Name), BC(BC) {}
+ MetadataRewriter(StringRef Name, RewriteInstance &RI);
+
+ std::optional<uint64_t> lookupSymbol(const StringRef Name);
public:
virtual ~MetadataRewriter() = default;
diff --git a/bolt/include/bolt/Rewrite/MetadataRewriters.h b/bolt/include/bolt/Rewrite/MetadataRewriters.h
index b71bd6cad2505..76face9888235 100644
--- a/bolt/include/bolt/Rewrite/MetadataRewriters.h
+++ b/bolt/include/bolt/Rewrite/MetadataRewriters.h
@@ -19,13 +19,13 @@ class BinaryContext;
// The list of rewriter build functions.
-std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createBuildIDRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createBuildIDRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createPseudoProbeRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createPseudoProbeRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createSDTRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createSDTRewriter(RewriteInstance &);
} // namespace bolt
} // namespace llvm
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 42094cb732107..92a2763e28c4d 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -42,6 +42,8 @@ class ProfileReaderBase;
/// optimizations) and rewriting. It also has the logic to coordinate such
/// events.
class RewriteInstance {
+ friend class MetadataRewriter;
+
public:
// This constructor has complex initialization that can fail during
// construction. Constructors can’t return errors, so clients must test \p Err
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index f9fc536f3569a..377de5a994a11 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2110,6 +2110,23 @@ ErrorOr<BinarySection &> BinaryContext::getSectionForAddress(uint64_t Address) {
return std::make_error_code(std::errc::bad_address);
}
+ErrorOr<BinarySection &>
+BinaryContext::getSectionForOutputAddress(uint64_t Address) {
+ for (auto &Sec : allocatableSections()) {
+ // Skip pseudo sections that serve a purpose of creating a corresponding
+ // entry in section header table
+ if (Sec.getOutputContents().empty())
+ continue;
+
+ uint64_t OutputAddress = Sec.getOutputAddress();
+ uint64_t OutputSize = Sec.getOutputSize();
+ if (OutputAddress && OutputAddress <= Address &&
+ Address < OutputAddress + OutputSize)
+ return Sec;
+ }
+ return std::make_error_code(std::errc::bad_address);
+}
+
ErrorOr<StringRef>
BinaryContext::getSectionNameForAddress(uint64_t Address) const {
if (ErrorOr<const BinarySection &> Section = getSectionForAddress(Address))
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 65e1032c579b5..d3ca951d7e453 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -85,7 +85,7 @@ void bolt::JumpTable::updateOriginal() {
uint64_t EntryOffset = BaseOffset;
for (MCSymbol *Entry : Entries) {
const uint64_t RelType =
- Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+ Type == JTT_NORMAL ? Relocation::getAbs64() : Relocation::getPC32();
const uint64_t RelAddend =
Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
// Replace existing relocation with the new one to allow any modifications
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index 7ff7a2288451c..b0aa40f9eac16 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -288,12 +288,17 @@ MCSymbol *MCPlusBuilder::getInstLabel(const MCInst &Inst) const {
}
MCSymbol *MCPlusBuilder::getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
- MCContext *Ctx) const {
+ MCContext *Ctx, bool Temp) const {
MCSymbol *Label = getInstLabel(Inst);
if (Label)
return Label;
- Label = Ctx->createNamedTempSymbol(Name);
+ if (Temp) {
+ Label = Ctx->createNamedTempSymbol(Name);
+ } else {
+ SmallVector<char, 16> Buf;
+ Label = Ctx->createLocalSymbol(Name.toStringRef(Buf));
+ }
setAnnotationOpValue(Inst, MCAnnotation::kLabel,
reinterpret_cast<int64_t>(Label));
return Label;
diff --git a/bolt/lib/Rewrite/BuildIDRewriter.cpp b/bolt/lib/Rewrite/BuildIDRewriter.cpp
index 83d0c9bfe182a..8a9c32619f6a9 100644
--- a/bolt/lib/Rewrite/BuildIDRewriter.cpp
+++ b/bolt/lib/Rewrite/BuildIDRewriter.cpp
@@ -39,8 +39,8 @@ class BuildIDRewriter final : public MetadataRewriter {
std::optional<uint64_t> BuildIDSize;
public:
- BuildIDRewriter(StringRef Name, BinaryContext &BC)
- : MetadataRewriter(Name, BC) {}
+ BuildIDRewriter(StringRef Name, RewriteInstance &RI)
+ : MetadataRewriter(Name, RI) {}
Error sectionInitializer() override;
@@ -108,6 +108,6 @@ Error BuildIDRewriter::postEmitFinalizer() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createBuildIDRewriter(BinaryContext &BC) {
- return std::make_unique<BuildIDRewriter>("build-id-rewriter", BC);
+llvm::bolt::createBuildIDRewriter(RewriteInstance &RI) {
+ return std::make_unique<BuildIDRewriter>("build-id-rewriter", RI);
}
diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index c83cf36982167..c403b8fcc33b2 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_library(LLVMBOLTRewrite
LinuxKernelRewriter.cpp
MachORewriteInstance.cpp
MetadataManager.cpp
+ MetadataRewriter.cpp
BuildIDRewriter.cpp
PseudoProbeRewriter.cpp
RewriteInstance.cpp
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 5a5e044184d0b..578ab40ec04ce 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -31,16 +31,6 @@ using namespace bolt;
namespace opts {
-static cl::opt<bool>
- AltInstHasPadLen("alt-inst-has-padlen",
- cl::desc("specify that .altinstructions has padlen field"),
- cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-
-static cl::opt<uint32_t>
- AltInstFeatureSize("alt-inst-feature-size",
- cl::desc("size of feature field in .altinstructions"),
- cl::init(2), cl::Hidden, cl::cat(BoltCategory));
-
static cl::opt<bool>
DumpAltInstructions("dump-alt-instructions",
cl::desc("dump Linux alternative instructions info"),
@@ -79,11 +69,6 @@ static cl::opt<bool>
cl::desc("dump Linux kernel static keys jump table"),
cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-static cl::opt<bool> LongJumpLabels(
- "long-jump-labels",
- cl::desc("always use long jumps/nops for Linux kernel static keys"),
- cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-
static cl::opt<bool>
PrintORC("print-orc",
cl::desc("print ORC unwind information for instructions"),
@@ -94,7 +79,7 @@ static cl::opt<bool>
/// Linux kernel version
struct LKVersion {
LKVersion() {}
- LKVersion(unsigned Major, unsigned Minor, unsigned Rev)
+ LKVersion(unsigned Major, unsigned Minor, unsigned Rev = 0)
: Major(Major), Minor(Minor), Rev(Rev) {}
bool operator<(const LKVersion &Other) const {
@@ -229,13 +214,16 @@ class LinuxKernelRewriter final : public MetadataRewriter {
static constexpr size_t STATIC_KEYS_JUMP_ENTRY_SIZE = 8;
struct JumpInfoEntry {
- bool Likely;
- bool InitValue;
+ bool Likely{false};
+ bool InitValue{false};
+ bool Nop{false};
+ MCSymbol *JumpInstLabel{nullptr};
+ BinaryFunction *BF{nullptr};
};
- SmallVector<JumpInfoEntry, 16> JumpInfo;
+ std::vector<JumpInfoEntry> JumpInfo;
- /// Static key entries that need nop conversion.
- DenseSet<uint32_t> NopIDs;
+ // Use long jumps/nops for Linux kernel static keys
+ bool LongJumpLabels{false};
/// Section containing static call table.
ErrorOr<BinarySection &> StaticCallSection = std::errc::bad_address;
@@ -249,11 +237,6 @@ class LinuxKernelRewriter final : public MetadataRewriter {
};
using StaticCallListType = std::vector<StaticCallInfo>;
StaticCallListType StaticCallEntries;
-
- /// Section containing the Linux exception table.
- ErrorOr<BinarySection &> ExceptionsSection = std::errc::bad_address;
- static constexpr size_t EXCEPTION_TABLE_ENTRY_SIZE = 12;
-
/// Functions with exception handling code.
DenseSet<BinaryFunction *> FunctionsWithExceptions;
@@ -266,6 +249,15 @@ class LinuxKernelRewriter final : public MetadataRewriter {
/// .altinstructions section.
ErrorOr<BinarySection &> AltInstrSection = std::errc::bad_address;
+ struct AltInstrEntry {
+ uint64_t Offset{0};
+ uint64_t OrgInstrAddr{0};
+ uint64_t AltInstrAddr{0};
+ uint8_t Instrlen{0};
+ uint8_t Replacementlen{0};
+ };
+ std::vector<AltInstrEntry> AltInstrEntries;
+
/// Section containing Linux bug table.
ErrorOr<BinarySection &> BugTableSection = std::errc::bad_address;
@@ -314,7 +306,7 @@ class LinuxKernelRewriter final : public MetadataRewriter {
Error readStaticCalls();
Error rewriteStaticCalls();
- Error readExceptionTable();
+ Error readExceptionTable(StringRef SectionName);
Error rewriteExceptionTable();
/// Paravirtual instruction patch sites.
@@ -332,8 +324,6 @@ class LinuxKernelRewriter final : public MetadataRewriter {
/// Handle alternative instruction info from .altinstructions.
Error readAltInstructions();
void processAltInstructionsPostCFG();
- Error tryReadAltInstructions(uint32_t AltInstFeatureSize,
- bool AltInstHasPadLen, bool ParseOnly);
/// Read .pci_fixup
Error readPCIFixupTable();
@@ -344,8 +334,8 @@ class LinuxKernelRewriter final : public MetadataRewriter {
Error updateStaticKeysJumpTablePostEmit();
public:
- LinuxKernelRewriter(BinaryContext &BC)
- : MetadataRewriter("linux-kernel-rewriter", BC) {}
+ LinuxKernelRewriter(RewriteInstance &RI)
+ : MetadataRewriter("linux-kernel-rewriter", RI) {}
Error preCFGInitializer() override {
if (Error E = detectLinuxKernelVersion())
@@ -359,7 +349,10 @@ class LinuxKernelRewriter final : public MetadataRewriter {
if (Error E = readStaticCalls())
return E;
- if (Error E = readExceptionTable())
+ if (Error E = readExceptionTable("__ex_table"))
+ return E;
+
+ if (Error E = readExceptionTable("__kvm_ex_table"))
return E;
if (Error E = readParaInstructions())
@@ -446,6 +439,8 @@ Error LinuxKernelRewriter::detectLinuxKernelVersion() {
LinuxKernelVersion = LKVersion(Major, Minor, Rev);
BC.outs() << "BOLT-INFO: Linux kernel version is " << Match[1].str()
<< "\n";
+ if (LinuxKernelVersion < LKVersion(5, 0))
+ return createStringError("Unsupported Linux kernel version");
return Error::success();
}
}
@@ -557,8 +552,8 @@ void LinuxKernelRewriter::processInstructionFixups() {
continue;
Fixup.Section.addRelocation(Fixup.Offset, &Fixup.Label,
- Fixup.IsPCRelative ? ELF::R_X86_64_PC32
- : ELF::R_X86_64_64,
+ Fixup.IsPCRelative ? Relocation::getPC32()
+ : Relocation::getAbs64(),
/*Addend*/ 0);
}
}
@@ -1074,7 +1069,7 @@ Error LinuxKernelRewriter::rewriteStaticCalls() {
StaticCallSection->getAddress() +
(Entry.ID - 1) * STATIC_CALL_ENTRY_SIZE;
StaticCallSection->addRelocation(EntryOffset, Entry.Label,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ Relocation::getPC32(), /*Addend*/ 0);
}
return Error::success();
@@ -1094,12 +1089,24 @@ Error LinuxKernelRewriter::rewriteStaticCalls() {
///
/// More info at:
/// https://www.kernel.org/doc/Documentation/x86/exception-tables.txt
-Error LinuxKernelRewriter::readExceptionTable() {
- ExceptionsSection = BC.getUniqueSectionByName("__ex_table");
+Error LinuxKernelRewriter::readExceptionTable(StringRef SectionName) {
+ ErrorOr<BinarySection &> ExceptionsSection =
+ BC.getUniqueSectionByName(SectionName);
if (!ExceptionsSection)
return Error::success();
- if (ExceptionsSection->getSize() % EXCEPTION_TABLE_ENTRY_SIZE)
+ size_t ExceptionTableEntrySize = 0;
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ ExceptionTableEntrySize = 12;
+ break;
+
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+ assert(ExceptionTableEntrySize && "exception table entry size is unknown");
+
+ if (ExceptionsSection->getSize() % ExceptionTableEntrySize)
return createStringError(errc::executable_format_error,
"exception table size error");
@@ -1111,7 +1118,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
while (Cursor && Cursor.tell() < ExceptionsSection->getSize()) {
const uint64_t InstAddress = AE.getPCRelAddress32(Cursor);
const uint64_t FixupAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t Data = AE.getU32(Cursor);
+ Cursor.seek(Cursor.tell() + ExceptionTableEntrySize - 8);
// Consume the status of the cursor.
if (!Cursor)
@@ -1125,8 +1132,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
if (opts::DumpExceptions) {
BC.outs() << "Exception Entry: " << EntryID << '\n';
BC.outs() << "\tInsn: 0x" << Twine::utohexstr(InstAddress) << '\n'
- << "\tFixup: 0x" << Twine::utohexstr(FixupAddress) << '\n'
- << "\tData: 0x" << Twine::utohexstr(Data) << '\n';
+ << "\tFixup: 0x" << Twine::utohexstr(FixupAddress) << '\n';
}
MCInst *Inst = nullptr;
@@ -1174,7 +1180,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
}
BC.outs() << "BOLT-INFO: parsed "
- << ExceptionsSection->getSize() / EXCEPTION_TABLE_ENTRY_SIZE
+ << ExceptionsSection->getSize() / ExceptionTableEntrySize
<< " exception table entries\n";
return Error::success();
@@ -1377,7 +1383,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
MCSymbol *Label =
BC.MIB->getOrCreateInstLabel(Inst, "__BUG_", BC.Ctx.get());
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, Label, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, Label,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1387,7 +1394,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
for (const uint32_t ID : FunctionBugList[&BF]) {
if (!EmittedIDs.count(ID)) {
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, nullptr, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, nullptr,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1399,95 +1407,69 @@ Error LinuxKernelRewriter::rewriteBugTable() {
/// The kernel can replace certain instruction sequences depending on hardware
/// it is running on and features specified during boot time. The information
/// about alternative instruction sequences is stored in .altinstructions
-/// section. The format of entries in this section is defined in
-/// arch/x86/include/asm/alternative.h:
-///
+/// section. The format of entries in this section is defined as
/// struct alt_instr {
/// s32 instr_offset;
/// s32 repl_offset;
-/// uXX feature;
+/// ...
/// u8 instrlen;
/// u8 replacementlen;
-/// u8 padlen; // present in older kernels
+/// ...
/// } __packed;
///
-/// Note that the structure is packed.
+/// Note that the structure is packed and field names may not be exactly the
+/// same.
///
-/// Since the size of the "feature" field could be either u16 or u32, and
-/// "padlen" presence is unknown, we attempt to parse .altinstructions section
-/// using all possible combinations (four at this time). Since we validate the
-/// contents of the section and its size, the detection works quite well.
-/// Still, we leave the user the opportunity to specify these features on the
-/// command line and skip the guesswork.
+/// To parse entries we only need to know the entry size and offset of
+/// the field 'instrlen'.
Error LinuxKernelRewriter::readAltInstructions() {
AltInstrSection = BC.getUniqueSectionByName(".altinstructions");
if (!AltInstrSection)
return Error::success();
- // Presence of "padlen" field.
- std::vector<bool> PadLenVariants;
- if (opts::AltInstHasPadLen.getNumOccurrences())
- PadLenVariants.push_back(opts::AltInstHasPadLen);
- else
- PadLenVariants = {false, true};
-
- // Size (in bytes) variants of "feature" field.
- std::vector<uint32_t> FeatureSizeVariants;
- if (opts::AltInstFeatureSize.getNumOccurrences())
- FeatureSizeVariants.push_back(opts::AltInstFeatureSize);
- else
- FeatureSizeVariants = {2, 4};
-
- for (bool AltInstHasPadLen : PadLenVariants) {
- for (uint32_t AltInstFeatureSize : FeatureSizeVariants) {
- LLVM_DEBUG({
- dbgs() << "BOLT-DEBUG: trying AltInstHasPadLen = " << AltInstHasPadLen
- << "; AltInstFeatureSize = " << AltInstFeatureSize << ";\n";
- });
- if (Error E = tryReadAltInstructions(AltInstFeatureSize, AltInstHasPadLen,
- /*ParseOnly*/ true)) {
- consumeError(std::move(E));
- continue;
- }
-
- LLVM_DEBUG(dbgs() << "Matched .altinstructions format\n");
-
- if (!opts::AltInstHasPadLen.getNumOccurrences())
- BC.outs() << "BOLT-INFO: setting --" << opts::AltInstHasPadLen.ArgStr
- << '=' << AltInstHasPadLen << '\n';
-
- if (!opts::AltInstFeatureSize.getNumOccurrences())
- BC.outs() << "BOLT-INFO: setting --" << opts::AltInstFeatureSize.ArgStr
- << '=' << AltInstFeatureSize << '\n';
-
- return tryReadAltInstructions(AltInstFeatureSize, AltInstHasPadLen,
- /*ParseOnly*/ false);
+ unsigned AltInstrEntrySize{0};
+ unsigned AltInstrEntryInstrlenOffset{0};
+
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (LinuxKernelVersion >= LKVersion(6, 3)) {
+ AltInstrEntrySize = 18;
+ AltInstrEntryInstrlenOffset = 16;
+ } else if (LinuxKernelVersion >= LKVersion(5, 10, 133)) {
+ AltInstrEntrySize = 12;
+ AltInstrEntryInstrlenOffset = 10;
+ } else {
+ AltInstrEntrySize = 13;
+ AltInstrEntryInstrlenOffset = 10;
}
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
}
- // We couldn't match the format. Read again to properly propagate the error
- // to the user.
- return tryReadAltInstructions(opts::AltInstFeatureSize,
- opts::AltInstHasPadLen, /*ParseOnly*/ false);
-}
+ BC.outs() << "BOLT-INFO: AltInstrEntrySize = " << AltInstrEntrySize
+ << ", AltInstrEntryInstrlenOffset = " << AltInstrEntryInstrlenOffset
+ << "\n";
-Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
- bool AltInstHasPadLen,
- bool ParseOnly) {
AddressExtractor AE(
AltInstrSection->getContents(), AltInstrSection->getAddress(),
BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
AddressExtractor::Cursor Cursor(0);
uint64_t EntryID = 0;
while (Cursor && !AE.eof(Cursor)) {
- const uint64_t OrgInstAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t AltInstAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t Feature = AE.getUnsigned(Cursor, AltInstFeatureSize);
- const uint8_t OrgSize = AE.getU8(Cursor);
- const uint8_t AltSize = AE.getU8(Cursor);
+ ++EntryID;
+ AltInstrEntries.push_back(AltInstrEntry());
+ AltInstrEntry &Entry = AltInstrEntries.back();
- // Older kernels may have the padlen field.
- const uint8_t PadLen = AltInstHasPadLen ? AE.getU8(Cursor) : 0;
+ Entry.Offset = Cursor.tell();
+ Entry.OrgInstrAddr = AE.getPCRelAddress32(Cursor);
+ Entry.AltInstrAddr = AE.getPCRelAddress32(Cursor);
+ Cursor.seek(Cursor.tell() + AltInstrEntryInstrlenOffset - 8);
+
+ Entry.Instrlen = AE.getU8(Cursor);
+ Entry.Replacementlen = AE.getU8(Cursor);
+ Cursor.seek(Cursor.tell() + AltInstrEntrySize -
+ (AltInstrEntryInstrlenOffset + 2));
if (!Cursor)
return createStringError(
@@ -1495,57 +1477,51 @@ Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
"out of bounds while reading .altinstructions: %s",
toString(Cursor.takeError()).c_str());
- ++EntryID;
-
if (opts::DumpAltInstructions) {
BC.outs() << "Alternative instruction entry: " << EntryID
- << "\n\tOrg: 0x" << Twine::utohexstr(OrgInstAddress)
- << "\n\tAlt: 0x" << Twine::utohexstr(AltInstAddress)
- << "\n\tFeature: 0x" << Twine::utohexstr(Feature)
- << "\n\tOrgSize: " << (int)OrgSize
- << "\n\tAltSize: " << (int)AltSize << '\n';
- if (AltInstHasPadLen)
- BC.outs() << "\tPadLen: " << (int)PadLen << '\n';
+ << "\n\tOrg: 0x" << Twine::utohexstr(Entry.OrgInstrAddr)
+ << "\n\tAlt: 0x" << Twine::utohexstr(Entry.AltInstrAddr)
+ << "\n\tInstrlen: " << (int)Entry.Instrlen
+ << "\n\tReplacementlen: " << (int)Entry.Replacementlen << '\n';
}
- if (AltSize > OrgSize)
+ if (Entry.Replacementlen > Entry.Instrlen)
return createStringError(errc::executable_format_error,
"error reading .altinstructions");
- BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(OrgInstAddress);
+ BinaryFunction *BF =
+ BC.getBinaryFunctionContainingAddress(Entry.OrgInstrAddr);
if (!BF && opts::Verbosity) {
BC.outs() << "BOLT-INFO: no function matches address 0x"
- << Twine::utohexstr(OrgInstAddress)
+ << Twine::utohexstr(Entry.OrgInstrAddr)
<< " of instruction from .altinstructions\n";
}
BinaryFunction *AltBF =
- BC.getBinaryFunctionContainingAddress(AltInstAddress);
- if (!ParseOnly && AltBF && BC.shouldEmit(*AltBF)) {
- BC.errs()
- << "BOLT-WARNING: alternative instruction sequence found in function "
- << *AltBF << '\n';
+ BC.getBinaryFunctionContainingAddress(Entry.AltInstrAddr);
+ if (AltBF) {
+ if (BC.isX86() &&
+ !AltBF->getOneName().starts_with(".altinstr_replacement"))
+ BC.errs() << "BOLT-WARNING: alternative instruction sequence found in "
+ "function "
+ << *AltBF << '\n';
AltBF->setIgnored();
}
if (!BF || !BF->hasInstructions())
continue;
- if (OrgInstAddress + OrgSize > BF->getAddress() + BF->getSize())
+ if (Entry.OrgInstrAddr + Entry.Instrlen > BF->getAddress() + BF->getSize())
return createStringError(errc::executable_format_error,
"error reading .altinstructions");
MCInst *Inst =
- BF->getInstructionAtOffset(OrgInstAddress - BF->getAddress());
+ BF->getInstructionAtOffset(Entry.OrgInstrAddr - BF->getAddress());
if (!Inst)
return createStringError(errc::executable_format_error,
"no instruction at address 0x%" PRIx64
" referenced by .altinstructions entry %d",
- OrgInstAddress, EntryID);
-
- if (ParseOnly)
- continue;
-
+ Entry.OrgInstrAddr, EntryID);
// There could be more than one alternative instruction sequences for the
// same original instruction. Annotate each alternative separately.
std::string AnnotationName = "AltInst";
@@ -1558,18 +1534,15 @@ Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
// Annotate all instructions from the original sequence. Note that it's not
// the most efficient way to look for instructions in the address range,
// but since alternative instructions are uncommon, it will do for now.
- for (uint32_t Offset = 1; Offset < OrgSize; ++Offset) {
- Inst = BF->getInstructionAtOffset(OrgInstAddress + Offset -
+ for (uint32_t Offset = 1; Offset < Entry.Instrlen; ++Offset) {
+ Inst = BF->getInstructionAtOffset(Entry.OrgInstrAddr + Offset -
BF->getAddress());
if (Inst)
BC.MIB->addAnnotation(*Inst, AnnotationName, EntryID);
}
}
-
- if (!ParseOnly)
- BC.outs() << "BOLT-INFO: parsed " << EntryID
- << " alternative instruction entries\n";
-
+ BC.outs() << "BOLT-INFO: parsed " << EntryID
+ << " alternative instruction entries\n";
return Error::success();
}
@@ -1691,6 +1664,8 @@ Error LinuxKernelRewriter::readPCIFixupTable() {
/// byte of the sequence with int3 before proceeding with actual code
/// replacement.
Error LinuxKernelRewriter::readStaticKeysJumpTable() {
+ LongJumpLabels = BC.isX86() && LinuxKernelVersion < LKVersion(5, 14);
+
const BinaryData *StaticKeysJumpTable =
BC.getBinaryDataByName("__start___jump_table");
if (!StaticKeysJumpTable)
@@ -1762,6 +1737,9 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
if (!BF || !BC.shouldEmit(*BF))
continue;
+ assert(BF->getOriginSection() &&
+ "the function did not originate from the file");
+ Info.BF = BF;
MCInst *Inst = BF->getInstructionAtOffset(JumpAddress - BF->getAddress());
if (!Inst)
@@ -1783,7 +1761,19 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
JumpAddress);
const uint64_t Size = BC.computeInstructionSize(*Inst);
- if (Size != 2 && Size != 5) {
+
+ auto checkSize = [this, Size]() {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (LongJumpLabels)
+ return Size == 5;
+ return Size == 2 || Size == 5;
+ default:
+ return false;
+ }
+ };
+
+ if (!checkSize()) {
return createStringError(
errc::executable_format_error,
"unexpected static keys jump size at address 0x%" PRIx64,
@@ -1805,7 +1795,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
// by the kernel patching code. Newer kernels can work with both short
// and long branches. The code for long conditional branch is larger
// than unconditional one, so we are pessimistic in our estimations.
- if (opts::LongJumpLabels)
+ if (LongJumpLabels)
BC.MIB->createLongCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get());
else
BC.MIB->createCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get());
@@ -1832,7 +1822,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
if (!BC.MIB->getOffset(*Inst))
BC.MIB->setOffset(*Inst, JumpAddress - BF->getAddress());
- if (opts::LongJumpLabels)
+ if (LongJumpLabels)
BC.MIB->setSize(*Inst, 5);
}
@@ -1865,21 +1855,27 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
const_cast<MCSymbol *>(BC.MIB->getTargetSymbol(Inst));
assert(Target && "Target symbol should be set.");
- const JumpInfoEntry &Info = JumpInfo[EntryID - 1];
+ JumpInfoEntry &Info = JumpInfo[EntryID - 1];
const bool IsBranch = Info.Likely ^ Info.InitValue;
uint32_t Size = *BC.MIB->getSize(Inst);
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
- llvm_unreachable("Wrong size for static keys jump instruction.");
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Size == 2)
+ ++NumShort;
+ else if (Size == 5)
+ ++NumLong;
+ else
+ llvm_unreachable("Wrong size for static keys jump instruction.");
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
MCInst NewInst;
// Replace the instruction with unconditional jump even if it needs to
// be nop in the binary.
- if (opts::LongJumpLabels) {
+ if (LongJumpLabels) {
BC.MIB->createLongUncondBranch(NewInst, Target, BC.Ctx.get());
} else {
// Newer kernels can handle short and long jumps for static keys.
@@ -1893,20 +1889,21 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
// Mark the instruction for nop conversion.
if (!IsBranch)
- NopIDs.insert(EntryID);
+ Info.Nop = true;
- MCSymbol *Label =
- BC.MIB->getOrCreateInstLabel(Inst, "__SK_", BC.Ctx.get());
+ Info.JumpInstLabel = BC.MIB->getOrCreateInstLabel(
+ Inst, formatv("__bolt.static_key_{0:X+8}_", EntryID), BC.Ctx.get(),
+ false);
// Create a relocation against the label.
const uint64_t EntryOffset = StaticKeysJumpTableAddress -
StaticKeysJumpSection->getAddress() +
(EntryID - 1) * 16;
- StaticKeysJumpSection->addRelocation(EntryOffset, Label,
- ELF::R_X86_64_PC32,
+ StaticKeysJumpSection->addRelocation(EntryOffset, Info.JumpInstLabel,
+ Relocation::getPC32(),
/*Addend*/ 0);
- StaticKeysJumpSection->addRelocation(EntryOffset + 4, Target,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ StaticKeysJumpSection->addRelocation(
+ EntryOffset + 4, Target, Relocation::getPC32(), /*Addend*/ 0);
}
}
}
@@ -1922,69 +1919,67 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
if (!StaticKeysJumpSection || !StaticKeysJumpSection->isFinalized())
return Error::success();
- const uint64_t SectionAddress = StaticKeysJumpSection->getAddress();
- AddressExtractor AE(StaticKeysJumpSection->getOutputContents(),
- SectionAddress, BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- AddressExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
- const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table");
- uint32_t EntryID = 0;
uint64_t NumShort = 0;
uint64_t NumLong = 0;
- while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) {
- const uint64_t JumpAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t TargetAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t KeyAddress = AE.getPCRelAddress64(Cursor);
-
- // Consume the status of the cursor.
- if (!Cursor)
- return createStringError(errc::executable_format_error,
- "out of bounds while updating static keys: %s",
- toString(Cursor.takeError()).c_str());
-
- ++EntryID;
-
- LLVM_DEBUG({
- dbgs() << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress)
- << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress)
- << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n';
- });
- (void)TargetAddress;
- (void)KeyAddress;
-
- BinaryFunction *BF =
- BC.getBinaryFunctionContainingAddress(JumpAddress,
- /*CheckPastEnd*/ false,
- /*UseMaxSize*/ true);
- assert(BF && "Cannot get function for modified static key.");
+ for (JumpInfoEntry &Info : JumpInfo) {
+ MCSymbol *Label = Info.JumpInstLabel;
+ if (!Label)
+ continue;
- if (!BF->isEmitted())
+ BinaryFunction *BF = Info.BF;
+ if (!BF || !BF->isEmitted())
continue;
- // Disassemble instruction to collect stats even if nop-conversion is
- // unnecessary.
- MutableArrayRef<uint8_t> Contents = MutableArrayRef<uint8_t>(
- reinterpret_cast<uint8_t *>(BF->getImageAddress()), BF->getImageSize());
- assert(Contents.size() && "Non-empty function image expected.");
+ std::optional<uint64_t> JumpAddress = lookupSymbol(Label->getName());
+ assert(JumpAddress && "missing static key jump instruction label");
+
+ uint64_t ContentsAddress{0};
+ uint64_t ContentsSize{0};
+ MutableArrayRef<uint8_t> Contents;
+
+ if (!BC.HasRelocations) {
+ const FunctionFragment *FF =
+ BF->getFunctionFragmentForOutputAddress(*JumpAddress);
+ assert(FF && "Can not get fragment for jump address");
+
+ ContentsAddress = FF->getAddress();
+ ContentsSize = FF->getImageSize();
+ Contents = MutableArrayRef<uint8_t>(FF->getOutputData(), ContentsSize);
+ } else {
+ ErrorOr<BinarySection &> Sec =
+ BC.getSectionForOutputAddress(*JumpAddress);
+ assert(Sec && "Can not get section for jump address.");
+
+ ContentsAddress = Sec->getOutputAddress();
+ ContentsSize = Sec->getOutputSize();
+ Contents = MutableArrayRef<uint8_t>(Sec->getOutputData(), ContentsSize);
+ }
MCInst Inst;
uint64_t Size;
- const uint64_t JumpOffset = JumpAddress - BF->getAddress();
+ const uint64_t JumpOffset = *JumpAddress - ContentsAddress;
if (!BC.DisAsm->getInstruction(Inst, Size, Contents.slice(JumpOffset), 0,
nulls())) {
llvm_unreachable("Unable to disassemble jump instruction.");
}
assert(BC.MIB->isBranch(Inst) && "Branch instruction expected.");
-
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
- llvm_unreachable("Unexpected size for static keys jump instruction.");
+ assert(JumpOffset + Size <= ContentsAddress + ContentsSize);
+
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Size == 2)
+ ++NumShort;
+ else if (Size == 5)
+ ++NumLong;
+ else
+ llvm_unreachable("Unexpected size for static keys jump instruction.");
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
// Check if we need to convert jump instruction into a nop.
- if (!NopIDs.contains(EntryID))
+ if (!Info.Nop)
continue;
SmallString<15> NopCode;
@@ -2003,6 +1998,6 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createLinuxKernelRewriter(BinaryContext &BC) {
- return std::make_unique<LinuxKernelRewriter>(BC);
+llvm::bolt::createLinuxKernelRewriter(RewriteInstance &RI) {
+ return std::make_unique<LinuxKernelRewriter>(RI);
}
diff --git a/bolt/lib/Rewrite/MetadataRewriter.cpp b/bolt/lib/Rewrite/MetadataRewriter.cpp
new file mode 100644
index 0000000000000..962e7704167b9
--- /dev/null
+++ b/bolt/lib/Rewrite/MetadataRewriter.cpp
@@ -0,0 +1,20 @@
+//===------------ bolt/Rewrite/MetadataRewriter.cpp -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Rewrite/MetadataRewriter.h"
+#include "bolt/Rewrite/RewriteInstance.h"
+
+using namespace llvm;
+using namespace bolt;
+
+MetadataRewriter::MetadataRewriter(StringRef Name, RewriteInstance &RI)
+ : Name(Name), RI(RI), BC(*RI.BC) {}
+
+std::optional<uint64_t> MetadataRewriter::lookupSymbol(const StringRef Name) {
+ return RI.Linker->lookupSymbol(Name);
+}
diff --git a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
index 9d6e914624a33..3e4d839b4cbc4 100644
--- a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
+++ b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
@@ -80,8 +80,8 @@ class PseudoProbeRewriter final : public MetadataRewriter {
std::shared_ptr<MCPseudoProbeDecoder> ProbeDecoderPtr;
public:
- PseudoProbeRewriter(BinaryContext &BC)
- : MetadataRewriter("pseudo-probe-rewriter", BC),
+ PseudoProbeRewriter(RewriteInstance &RI)
+ : MetadataRewriter("pseudo-probe-rewriter", RI),
ProbeDecoderPtr(std::make_shared<MCPseudoProbeDecoder>()) {
BC.setPseudoProbeDecoder(ProbeDecoderPtr);
}
@@ -447,6 +447,6 @@ void PseudoProbeRewriter::encodePseudoProbes() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createPseudoProbeRewriter(BinaryContext &BC) {
- return std::make_unique<PseudoProbeRewriter>(BC);
+llvm::bolt::createPseudoProbeRewriter(RewriteInstance &RI) {
+ return std::make_unique<PseudoProbeRewriter>(RI);
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 70a9f084f009b..4b161af262209 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -541,9 +541,13 @@ Error RewriteInstance::discoverStorage() {
BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
Phdr.p_vaddr, Phdr.p_memsz, Phdr.p_offset,
Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
- if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
- Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
- BC->IsLinuxKernel = true;
+ switch (BC->TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
+ BC->IsLinuxKernel = true;
+ break;
+ default:;
+ }
break;
case ELF::PT_INTERP:
BC->HasInterpHeader = true;
@@ -3208,13 +3212,13 @@ void RewriteInstance::preprocessProfileData() {
void RewriteInstance::initializeMetadataManager() {
if (BC->IsLinuxKernel)
- MetadataManager.registerRewriter(createLinuxKernelRewriter(*BC));
+ MetadataManager.registerRewriter(createLinuxKernelRewriter(*this));
- MetadataManager.registerRewriter(createBuildIDRewriter(*BC));
+ MetadataManager.registerRewriter(createBuildIDRewriter(*this));
- MetadataManager.registerRewriter(createPseudoProbeRewriter(*BC));
+ MetadataManager.registerRewriter(createPseudoProbeRewriter(*this));
- MetadataManager.registerRewriter(createSDTRewriter(*BC));
+ MetadataManager.registerRewriter(createSDTRewriter(*this));
}
void RewriteInstance::processSectionMetadata() {
@@ -3878,6 +3882,7 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
<< " to 0x" << Twine::utohexstr(Function.getAddress())
<< '\n');
MapSection(*FuncSection, Function.getAddress());
+ Function.getLayout().getMainFragment().setAddress(Function.getAddress());
Function.setImageAddress(FuncSection->getAllocAddress());
Function.setImageSize(FuncSection->getOutputSize());
assert(Function.getImageSize() <= Function.getMaxSize() &&
diff --git a/bolt/lib/Rewrite/SDTRewriter.cpp b/bolt/lib/Rewrite/SDTRewriter.cpp
index a3928c554ad66..2558403fac763 100644
--- a/bolt/lib/Rewrite/SDTRewriter.cpp
+++ b/bolt/lib/Rewrite/SDTRewriter.cpp
@@ -55,7 +55,8 @@ class SDTRewriter final : public MetadataRewriter {
void printSDTMarkers() const;
public:
- SDTRewriter(StringRef Name, BinaryContext &BC) : MetadataRewriter(Name, BC) {}
+ SDTRewriter(StringRef Name, RewriteInstance &RI)
+ : MetadataRewriter(Name, RI) {}
Error preCFGInitializer() override;
@@ -173,6 +174,6 @@ void SDTRewriter::printSDTMarkers() const {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createSDTRewriter(BinaryContext &BC) {
- return std::make_unique<SDTRewriter>("sdt-rewriter", BC);
+llvm::bolt::createSDTRewriter(RewriteInstance &RI) {
+ return std::make_unique<SDTRewriter>("sdt-rewriter", RI);
}
diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s
index 83d2cd0634d08..3e299685cf5bb 100644
--- a/bolt/test/X86/linux-alt-instruction.s
+++ b/bolt/test/X86/linux-alt-instruction.s
@@ -6,31 +6,9 @@
# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-cfg --alt-inst-feature-size=2 -o %t.out \
+# RUN: llvm-bolt %t.exe --print-cfg -o %t.out \
# RUN: | FileCheck %s
-## Older kernels used to have padlen field in alt_instr. Check compatibility.
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown --defsym PADLEN=1 \
-# RUN: %s -o %t.padlen.o
-# RUN: %clang %cflags -nostdlib %t.padlen.o -o %t.padlen.exe \
-# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.padlen.exe --print-cfg --alt-inst-has-padlen -o %t.padlen.out \
-# RUN: | FileCheck %s
-
-## Check with a larger size of "feature" field in alt_instr.
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown \
-# RUN: --defsym FEATURE_SIZE_4=1 %s -o %t.fs4.o
-# RUN: %clang %cflags -nostdlib %t.fs4.o -o %t.fs4.exe \
-# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.fs4.exe --print-cfg --alt-inst-feature-size=4 -o %t.fs4.out \
-# RUN: | FileCheck %s
-
-## Check that out-of-bounds read is handled properly.
-
-# RUN: not llvm-bolt %t.fs4.exe --alt-inst-feature-size=2 -o %t.fs4.out
-
## Check that BOLT automatically detects structure fields in .altinstructions.
# RUN: llvm-bolt %t.exe --print-cfg -o %t.out | FileCheck %s
@@ -78,11 +56,7 @@ _start:
.long .L0 - . # org instruction
.long .A0 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x72 # feature flags
-.else
.word 0x72 # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .A1 - .A0 # alt size
.ifdef PADLEN
@@ -91,11 +65,7 @@ _start:
.long .L0 - . # org instruction
.long .A1 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x3b # feature flags
-.else
.word 0x3b # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .A2 - .A1 # alt size
.ifdef PADLEN
@@ -104,11 +74,7 @@ _start:
.long .L0 - . # org instruction
.long .A2 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x110 # feature flags
-.else
.word 0x110 # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .Ae - .A2 # alt size
.ifdef PADLEN
@@ -148,7 +114,7 @@ _start:
.globl linux_banner
.type linux_banner, @object
linux_banner:
- .string "Linux version 6.6.61\n"
+ .string "Linux version 6.1\n"
.size linux_banner, . - linux_banner
## Fake Linux Kernel sections.
>From 5fd581eccaebaf7d7f3602c5823f3da0b51fec38 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Sun, 9 Mar 2025 18:27:49 +0800
Subject: [PATCH 2/6] [BOLT][Linux] Add support for AArch64
Basic support for AArch64
---
bolt/include/bolt/Core/BinaryContext.h | 3 +++
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 27 ++++++++++++++++++-
bolt/lib/Rewrite/RewriteInstance.cpp | 23 ++++++++++++----
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 10 +++++++
4 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 77104d4c2f9cc..959dd11e4a799 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -613,6 +613,9 @@ class BinaryContext {
/// Addresses reserved for kernel on x86_64 start at this location.
static constexpr uint64_t KernelStartX86_64 = 0xFFFF'FFFF'8000'0000;
+ /// Addresses reserved for kernel on aarch64 start at this location.
+ static constexpr uint64_t KernelStartAArch64 = 0xFFFF'0000'0000'0000;
+
/// Map address to a constant island owner (constant data in code section)
std::map<uint64_t, BinaryFunction *> AddressToConstantIslandMap;
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 578ab40ec04ce..6c02e1784afe9 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -1101,6 +1101,13 @@ Error LinuxKernelRewriter::readExceptionTable(StringRef SectionName) {
ExceptionTableEntrySize = 12;
break;
+ case llvm::Triple::aarch64:
+ if (LinuxKernelVersion >= LKVersion(5, 16))
+ ExceptionTableEntrySize = 12;
+ else
+ ExceptionTableEntrySize = 8;
+ break;
+
default:
llvm_unreachable("Unsupported architecture");
}
@@ -1414,7 +1421,7 @@ Error LinuxKernelRewriter::rewriteBugTable() {
/// ...
/// u8 instrlen;
/// u8 replacementlen;
-/// ...
+/// ...
/// } __packed;
///
/// Note that the structure is packed and field names may not be exactly the
@@ -1443,6 +1450,10 @@ Error LinuxKernelRewriter::readAltInstructions() {
AltInstrEntryInstrlenOffset = 10;
}
break;
+ case llvm::Triple::aarch64:
+ AltInstrEntrySize = 12;
+ AltInstrEntryInstrlenOffset = 10;
+ break;
default:
llvm_unreachable("Unsupported architecture");
}
@@ -1768,6 +1779,8 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
if (LongJumpLabels)
return Size == 5;
return Size == 2 || Size == 5;
+ case llvm::Triple::aarch64:
+ return Size == 4;
default:
return false;
}
@@ -1868,6 +1881,12 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
else
llvm_unreachable("Wrong size for static keys jump instruction.");
break;
+ case llvm::Triple::aarch64:
+ if (Size == 4)
+ ++NumLong;
+ else
+ llvm_unreachable("Wrong size for static keys jump instruction.");
+ break;
default:
llvm_unreachable("Unsupported architecture");
}
@@ -1974,6 +1993,12 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
else
llvm_unreachable("Unexpected size for static keys jump instruction.");
break;
+ case llvm::Triple::aarch64:
+ if (Size == 4)
+ ++NumLong;
+ else
+ llvm_unreachable("Unexpected size for static keys jump instruction.");
+ break;
default:
llvm_unreachable("Unsupported architecture");
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 4b161af262209..3a7e76778eb2d 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -546,6 +546,10 @@ Error RewriteInstance::discoverStorage() {
if (Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
BC->IsLinuxKernel = true;
break;
+ case llvm::Triple::aarch64:
+ if (Phdr.p_vaddr >= BinaryContext::KernelStartAArch64)
+ BC->IsLinuxKernel = true;
+ break;
default:;
}
break;
@@ -1040,12 +1044,11 @@ void RewriteInstance::discoverFileObjects() {
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName
<< " for function\n");
- if (SymbolAddress == Section->getAddress() + Section->getSize()) {
+ if (SymbolAddress >= Section->getAddress() + Section->getSize()) {
assert(SymbolSize == 0 &&
- "unexpect non-zero sized symbol at end of section");
+ "unexpect non-zero sized symbol outside section");
LLVM_DEBUG(
- dbgs()
- << "BOLT-DEBUG: rejecting as symbol points to end of its section\n");
+ dbgs() << "BOLT-DEBUG: rejecting as symbol is outside its section\n");
registerName(SymbolSize);
continue;
}
@@ -5700,8 +5703,18 @@ void RewriteInstance::rewriteFile() {
OS.pwrite(reinterpret_cast<char *>(Function->getImageAddress()),
Function->getImageSize(), Function->getFileOffset());
+ bool ShouldWriteNops = true;
+
+ // For AArch64, Linux kernel alternative instruction replacement sequences
+ // are not in a seperate section as for X86, but reside in gaps between
+ // functions.
+ // Avoid overwriting them by skipping writing nops here.
+ if (BC->IsLinuxKernel && BC->isAArch64() && !BC->HasRelocations)
+ ShouldWriteNops = false;
+
// Write nops at the end of the function.
- if (Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {
+ if (ShouldWriteNops &&
+ Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {
uint64_t Pos = OS.tell();
OS.seek(Function->getFileOffset() + Function->getImageSize());
BC->MAB->writeNopData(
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index f79d5a747246e..ae5df0c02f056 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -1896,6 +1896,16 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
*Ctx, 0)));
}
+ void createCondBranch(MCInst &Inst, const MCSymbol *TBB, unsigned CC,
+ MCContext *Ctx) const override {
+ Inst.setOpcode(AArch64::Bcc);
+ Inst.clear();
+ Inst.addOperand(MCOperand::createImm(CC));
+ Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
+ Inst, MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx),
+ *Ctx, 0)));
+ }
+
bool shouldRecordCodeRelocation(uint64_t RelType) const override {
switch (RelType) {
case ELF::R_AARCH64_ABS64:
>From 4fd0fca0dc2983de7c9d90dece5f7713cdfd1522 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Sun, 9 Mar 2025 18:06:10 +0800
Subject: [PATCH 3/6] [BOLT][Linux] Skip some functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Skip some functions to simplify things, making BOLT for Linux more reliable
and saving development efforts.
**skip functions defined in assembly code**
We use "-bolt-function-list-file" to gather a list of C function when building
Linux kernel, then BOLT can choose to optimize them only.
BOLT can not handle some functions defined in assembly code reliably, since
they may have extra semantics/enforcements BOLT can never know. For example,
irq_entries_start defined in arch/x86/include/asm/idtentry.h is actually an
“array” but BOLT views it as an ordinary function. If BOLT applies basic
block reordering, instrumentation, etc to it, run time errors would happen.
We could explicitly specify those functions to skip, but to find all of them
we usually need to test & debug many times. That is a lot of work and we may
still miss some.
In my own experience, when adapting BOLT for Linux to a new architecture or
Linux version, we may spend lots of time on fixing runtime errors related to
functions defined in assembly code.
Only handling C functions makes BOLT for Linux more reliable, and may save
lots of development efforts.
"-bolt-function-list-file" can be used as follows:
* put "-mllvm -bolt-function-list-file=filename" in KCFLAGS when
buildling Linux
* specify "-funcs-file-no-regex=filename" when invoking llvm-bolt
**skip functions that should keep their address**
Some functions's address are used in add/sub/comparision. To make things
simple, they should keep their address (after relocation mode is
supported).
See __bpf_call_base in kernel/bpf/core.c for an example.
Currently, we just skip them.
"bolt-keep-address-function-list-file" can be used as follows:
* put "-mllvm -bolt-keep-address-function-list-file=filename" in
KCFLAGS when buildling Linux
* specify "-keep-address-funcs-file-no-regex=filename" when
invoking llvm-bolt
**skip functions not in .text**
Functions in sections other than .text (e.g. .head.text, .init.text,
.exit.text) are (mostly) run during initialization and shutdown, and
not (much) relevant to application performance.
Skipping them also helps to avoid runtime errors, especially those
even before the first message is printed out, which are not easy to
debug.
Only handling functions in .text also make sure no function is moved to
a different section (after relocation mode is supported). Linux kernel
code may compare function pointers with section boundary symbols, and
if we move functions to another section, runtime errors may happen.
---
bolt/include/bolt/Core/BinaryContext.h | 7 ++++
bolt/include/bolt/Core/BinaryFunction.h | 6 ++++
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 27 ++++++++++++++
bolt/lib/Rewrite/RewriteInstance.cpp | 23 +++++++++++-
llvm/lib/CodeGen/CodeGenPrepare.cpp | 45 ++++++++++++++++++++++++
5 files changed, 107 insertions(+), 1 deletion(-)
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 959dd11e4a799..94ae66e48c749 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -433,6 +433,13 @@ class BinaryContext {
Address);
}
+ bool isInRange(StringRef NameStart, StringRef NameEnd,
+ uint64_t Address) const {
+ ErrorOr<uint64_t> Start = getSymbolValue(NameStart);
+ ErrorOr<uint64_t> End = getSymbolValue(NameEnd);
+ return Start && End && *Start <= Address && Address < *End;
+ }
+
/// Return size of an entry for the given jump table \p Type.
uint64_t getJumpTableEntrySize(JumpTable::JumpTableType Type) const {
return Type == JumpTable::JTT_PIC ? 4 : AsmInfo->getCodePointerSize();
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 20caebe4b129c..3cd2ca8b223df 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -294,6 +294,9 @@ class BinaryFunction {
/// Pseudo functions should not be disassembled or emitted.
bool IsPseudo{false};
+ // True if address of this function can not be changed
+ bool KeepAddress{false};
+
/// True if the original function code has all necessary relocations to track
/// addresses of functions emitted to new locations. Typically set for
/// functions that we are not going to emit.
@@ -1329,6 +1332,9 @@ class BinaryFunction {
/// otherwise processed.
bool isPseudo() const { return IsPseudo; }
+ /// Return true if address of this function can not be changed
+ bool mustKeepAddress() const { return KeepAddress; }
+
/// Return true if the function contains explicit or implicit indirect branch
/// to its split fragments, e.g., split jump table, landing pad in split
/// fragment.
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 6c02e1784afe9..f49761f229ba6 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -341,6 +341,33 @@ class LinuxKernelRewriter final : public MetadataRewriter {
if (Error E = detectLinuxKernelVersion())
return E;
+ auto ShouldIgnore = [this](const BinaryFunction &Function) {
+ std::optional<StringRef> SectionName = Function.getOriginSectionName();
+ if (!SectionName || *SectionName != ".text")
+ return true;
+
+ uint64_t Address = Function.getAddress();
+
+ if (BC.isX86()) {
+ BinaryData *BDStart = BC.getBinaryDataByName("irq_entries_start");
+ if (BDStart && BDStart->containsAddress(Address))
+ return true;
+
+ if (BC.isInRange("__static_call_text_start", "__static_call_text_end",
+ Address))
+ return true;
+ }
+
+ if (BC.isInRange("__noinstr_text_start", "__noinstr_text_end", Address))
+ return true;
+
+ return false;
+ };
+
+ for (BinaryFunction *Function : BC.getAllBinaryFunctions())
+ if (ShouldIgnore(*Function))
+ Function->setIgnored();
+
processLKSections();
if (Error E = processSMPLocks())
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 3a7e76778eb2d..e408fa5065a90 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -137,6 +137,16 @@ static cl::opt<std::string> FunctionNamesFileNR(
cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,
cl::cat(BoltCategory));
+static cl::list<std::string> KeepAddressFunctionNamesNR(
+ "keep-address-funcs-no-regex", cl::CommaSeparated,
+ cl::desc("KeepAddress functions from the list (non-regex)"),
+ cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));
+
+static cl::opt<std::string> KeepAddressFunctionNamesFileNR(
+ "keep-address-funcs-file-no-regex",
+ cl::desc("file with list of KeepAddress functions to optimize (non-regex)"),
+ cl::Hidden, cl::cat(BoltCategory));
+
cl::opt<bool>
KeepTmp("keep-tmp",
cl::desc("preserve intermediate .o file"),
@@ -2989,6 +2999,8 @@ void RewriteInstance::selectFunctionsToProcess() {
populateFunctionNames(opts::FunctionNamesFile, opts::ForceFunctionNames);
populateFunctionNames(opts::SkipFunctionNamesFile, opts::SkipFunctionNames);
populateFunctionNames(opts::FunctionNamesFileNR, opts::ForceFunctionNamesNR);
+ populateFunctionNames(opts::KeepAddressFunctionNamesFileNR,
+ opts::KeepAddressFunctionNamesNR);
// Make a set of functions to process to speed up lookups.
std::unordered_set<std::string> ForceFunctionsNR(
@@ -3003,6 +3015,10 @@ void RewriteInstance::selectFunctionsToProcess() {
exit(1);
}
+ std::unordered_set<std::string> KeepAddressFunctionsNR(
+ opts::KeepAddressFunctionNamesNR.begin(),
+ opts::KeepAddressFunctionNamesNR.end());
+
uint64_t LiteThresholdExecCount = 0;
if (opts::LiteThresholdPct) {
if (opts::LiteThresholdPct > 100)
@@ -3050,7 +3066,8 @@ void RewriteInstance::selectFunctionsToProcess() {
for (std::string &Name : opts::SkipFunctionNames)
if (Function.hasNameRegex(Name))
return true;
-
+ if (BC->HasRelocations && Function.mustKeepAddress())
+ return true;
return false;
};
@@ -3101,6 +3118,10 @@ void RewriteInstance::selectFunctionsToProcess() {
for (auto &BFI : BC->getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
+ for (const StringRef Name : Function.getNames())
+ if (KeepAddressFunctionsNR.count(Name.str()))
+ Function.KeepAddress = true;
+
// Pseudo functions are explicitly marked by us not to be processed.
if (Function.isPseudo()) {
Function.IsIgnored = true;
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 088062afab177..09242fe1cb602 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -85,6 +85,7 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
@@ -274,6 +275,14 @@ static cl::opt<bool>
DisableDeletePHIs("disable-cgp-delete-phis", cl::Hidden, cl::init(false),
cl::desc("Disable elimination of dead PHI nodes."));
+cl::opt<std::string>
+ BoltFunctionListFile("bolt-function-list-file", cl::Hidden,
+ cl::desc("Specify BOLT function list file"));
+
+cl::opt<std::string> BoltKeepAddressFunctionListFile(
+ "bolt-keep-address-function-list-file", cl::Hidden,
+ cl::desc("Specify BOLT KeepAddress function list file"));
+
namespace {
enum ExtType {
@@ -505,7 +514,43 @@ class CodeGenPrepareLegacyPass : public FunctionPass {
char CodeGenPrepareLegacyPass::ID = 0;
+template <typename T> void GatherForBoltKA(raw_fd_ostream &OS, T &I) {
+ switch (I.getOpcode()) {
+ case Instruction::ICmp:
+ case Instruction::PtrToInt:
+ for (Use &U : I.operands())
+ if (auto *FF = dyn_cast<Function>(U.get()))
+ OS << FF->getName() << "\n";
+ break;
+ default:;
+ }
+ for (Use &U : I.operands())
+ if (auto *CE = dyn_cast<ConstantExpr>(U.get()))
+ GatherForBoltKA(OS, *CE);
+}
+
bool CodeGenPrepareLegacyPass::runOnFunction(Function &F) {
+ if (!BoltFunctionListFile.empty()) {
+ std::error_code EC;
+ raw_fd_ostream OS(BoltFunctionListFile, EC, sys::fs::OpenFlags::OF_Append);
+ if (EC)
+ report_fatal_error(Twine(BoltFunctionListFile) + ": " + EC.message());
+ OS << F.getName() << "\n";
+ }
+
+ if (!BoltKeepAddressFunctionListFile.empty()) {
+ std::error_code EC;
+ raw_fd_ostream OS(BoltKeepAddressFunctionListFile, EC,
+ sys::fs::OpenFlags::OF_Append);
+ if (EC)
+ report_fatal_error(Twine(BoltKeepAddressFunctionListFile) + ": " +
+ EC.message());
+
+ for (BasicBlock &BB : F)
+ for (Instruction &I : BB)
+ GatherForBoltKA(OS, I);
+ }
+
if (skipFunction(F))
return false;
auto TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
>From 176e30f666c8632ce367540c40bce84abdaf4651 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Wed, 12 Mar 2025 00:22:16 +0800
Subject: [PATCH 4/6] [BOLT] Fix errors of undefined symbol or out of range
fixup
Make sure an instruction's label is added to undefinedSymbols before it is
removed
---
bolt/include/bolt/Core/BinaryBasicBlock.h | 11 +++++++++++
bolt/include/bolt/Core/BinaryContext.h | 5 +++++
bolt/include/bolt/Core/BinaryFunction.h | 15 +++++++++++++++
bolt/lib/Core/BinaryBasicBlock.cpp | 5 +++++
bolt/lib/Core/BinaryEmitter.cpp | 4 ++++
bolt/lib/Core/BinaryFunction.cpp | 10 ++--------
bolt/lib/Core/FunctionLayout.cpp | 4 ++++
7 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h
index 25cccc4edecf6..5334f7e02fcb7 100644
--- a/bolt/include/bolt/Core/BinaryBasicBlock.h
+++ b/bolt/include/bolt/Core/BinaryBasicBlock.h
@@ -690,10 +690,16 @@ class BinaryBasicBlock {
void setCanOutline(const bool Flag) { CanOutline = Flag; }
+ void undefineLabels() const {
+ for (const MCInst &Inst : Instructions)
+ undefineInstLabel(Inst);
+ }
+
/// Erase pseudo instruction at a given iterator.
/// Return iterator following the removed instruction.
iterator erasePseudoInstruction(iterator II) {
--NumPseudos;
+ undefineInstLabel(*II);
return Instructions.erase(II);
}
@@ -701,6 +707,7 @@ class BinaryBasicBlock {
/// Return iterator following the removed instruction.
iterator eraseInstruction(iterator II) {
adjustNumPseudos(*II, -1);
+ undefineInstLabel(*II);
return Instructions.erase(II);
}
@@ -718,6 +725,7 @@ class BinaryBasicBlock {
/// Erase all instructions.
void clear() {
+ undefineLabels();
Instructions.clear();
NumPseudos = 0;
}
@@ -742,6 +750,7 @@ class BinaryBasicBlock {
adjustNumPseudos(Begin, End, 1);
auto I = II - Instructions.begin();
+ undefineInstLabel(*II);
Instructions.insert(Instructions.erase(II), Begin, End);
return I + Instructions.begin();
}
@@ -917,6 +926,8 @@ class BinaryBasicBlock {
uint64_t getHash() const { return Hash; }
private:
+ void undefineInstLabel(const llvm::MCInst &Inst) const;
+
void adjustNumPseudos(const MCInst &Inst, int Sign);
template <typename Itr> void adjustNumPseudos(Itr Begin, Itr End, int Sign) {
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 94ae66e48c749..bbfda368fb605 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -552,6 +552,11 @@ class BinaryContext {
/// binary and functions created by BOLT.
std::vector<BinaryFunction *> getAllBinaryFunctions();
+ void undefineInstLabel(const MCInst &Inst) const {
+ if (MCSymbol *const Label = MIB->getInstLabel(Inst))
+ UndefinedSymbols.insert(Label);
+ }
+
/// Construct a jump table for \p Function at \p Address or return an existing
/// one at that location.
///
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 3cd2ca8b223df..ef43183aa9e4e 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1201,6 +1201,21 @@ class BinaryFunction {
/// Return true if all callbacks returned true, false otherwise.
bool forEachEntryPoint(EntryPointCallbackTy Callback) const;
+ void undefineLabels() {
+ for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
+ BC.UndefinedSymbols.insert(LI.second);
+
+ for (MCSymbol *const EndLabel : FunctionEndLabels)
+ if (EndLabel)
+ BC.UndefinedSymbols.insert(EndLabel);
+
+ for (const std::pair<const uint32_t, MCInst> &II : Instructions)
+ BC.undefineInstLabel(II.second);
+
+ for (const BinaryBasicBlock *BB : BasicBlocks)
+ BB->undefineLabels();
+ }
+
/// Return MC symbol associated with the end of the function.
MCSymbol *
getFunctionEndLabel(const FragmentNum Fragment = FragmentNum::main()) const {
diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp
index 2a2192b79bb4b..83b477be02d27 100644
--- a/bolt/lib/Core/BinaryBasicBlock.cpp
+++ b/bolt/lib/Core/BinaryBasicBlock.cpp
@@ -44,6 +44,11 @@ const JumpTable *BinaryBasicBlock::getJumpTable() const {
return JT;
}
+void BinaryBasicBlock::undefineInstLabel(const llvm::MCInst &Inst) const {
+ BinaryContext &BC = Function->getBinaryContext();
+ BC.undefineInstLabel(Inst);
+}
+
void BinaryBasicBlock::adjustNumPseudos(const MCInst &Inst, int Sign) {
BinaryContext &BC = Function->getBinaryContext();
if (BC.MIB->isPseudo(Inst))
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 1aad25242712f..50b8bd8f8e343 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -287,6 +287,10 @@ void BinaryEmitter::emitFunctions() {
// Emit functions added by BOLT.
emit(BC.getInjectedBinaryFunctions());
+ for (BinaryFunction *BF : SortedFunctions)
+ if (!BF->isEmitted())
+ BF->undefineLabels();
+
// Mark the end of hot text.
if (opts::HotText) {
if (BC.HasWarmSection)
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index ff5eb5cf6e1eb..3e7e8f6f861ed 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -3047,17 +3047,11 @@ uint64_t BinaryFunction::getInstructionCount() const {
}
void BinaryFunction::clearDisasmState() {
+ undefineLabels();
+
clearList(Instructions);
clearList(IgnoredBranches);
clearList(TakenBranches);
-
- if (BC.HasRelocations) {
- for (std::pair<const uint32_t, MCSymbol *> &LI : Labels)
- BC.UndefinedSymbols.insert(LI.second);
- for (MCSymbol *const EndLabel : FunctionEndLabels)
- if (EndLabel)
- BC.UndefinedSymbols.insert(EndLabel);
- }
}
void BinaryFunction::setTrapOnEntry() {
diff --git a/bolt/lib/Core/FunctionLayout.cpp b/bolt/lib/Core/FunctionLayout.cpp
index 4498fc44da954..1f1151a708f84 100644
--- a/bolt/lib/Core/FunctionLayout.cpp
+++ b/bolt/lib/Core/FunctionLayout.cpp
@@ -150,6 +150,10 @@ void FunctionLayout::eraseBasicBlocks(
FF.StartIndex -= TotalErased;
TotalErased += Erased;
}
+ for (BinaryBasicBlock *BB : Blocks) {
+ if (IsErased(BB))
+ BB->undefineLabels();
+ }
llvm::erase_if(Blocks, IsErased);
// Remove empty fragments at the end
>From 2658e746f41afe980b8c70c2ad1725aca994f031 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Wed, 12 Mar 2025 00:44:18 +0800
Subject: [PATCH 5/6] [BOLT][Linux] Add support for relocation mode
---
bolt/include/bolt/Core/BinarySection.h | 4 +
bolt/include/bolt/Core/MCPlusBuilder.h | 6 ++
bolt/include/bolt/Core/Relocation.h | 7 ++
bolt/include/bolt/Passes/PatchEntries.h | 6 ++
bolt/include/bolt/Rewrite/RewriteInstance.h | 3 +
bolt/lib/Core/BinaryContext.cpp | 2 +-
bolt/lib/Core/BinaryEmitter.cpp | 4 +-
bolt/lib/Core/BinarySection.cpp | 12 +++
bolt/lib/Core/Relocation.cpp | 13 +++
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 26 +++--
bolt/lib/Rewrite/RewriteInstance.cpp | 101 +++++++++++++++-----
11 files changed, 141 insertions(+), 43 deletions(-)
diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h
index 9c252c3675951..31f0b9de39af8 100644
--- a/bolt/include/bolt/Core/BinarySection.h
+++ b/bolt/include/bolt/Core/BinarySection.h
@@ -385,6 +385,10 @@ class BinarySection {
Patches.emplace_back(BinaryPatch(Offset, Bytes));
}
+ void addPatch(uint64_t Offset, StringRef Bytes) {
+ addPatch(Offset, SmallVector<char>(Bytes.begin(), Bytes.end()));
+ }
+
/// Register patcher for this section.
void registerPatcher(std::unique_ptr<BinaryPatcher> BPatcher) {
Patcher = std::move(BPatcher);
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 52643ffcd5b78..dadc627c6c6bb 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -689,6 +689,12 @@ class MCPlusBuilder {
return StringRef();
}
+ /// Used to fill the executable space with undefined instructions.
+ virtual StringRef getUndefFillValue() const {
+ llvm_unreachable("not implemented");
+ return StringRef();
+ }
+
/// Interface and basic functionality of a MCInstMatcher. The idea is to make
/// it easy to match one or more MCInsts against a tree-like pattern and
/// extract the fragment operands. Example:
diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h
index 933f62a31f8fd..5bb8e2c569c99 100644
--- a/bolt/include/bolt/Core/Relocation.h
+++ b/bolt/include/bolt/Core/Relocation.h
@@ -92,6 +92,9 @@ struct Relocation {
/// Return true if relocation type is RELATIVE
static bool isRelative(uint64_t Type);
+ /// Return true if relocation type is GLOB_DAT
+ static bool isGlobDat(uint64_t Type);
+
/// Return true if relocation type is IRELATIVE
static bool isIRelative(uint64_t Type);
@@ -124,6 +127,10 @@ struct Relocation {
/// otherwise.
bool isRelative() const { return isRelative(Type); }
+ /// Return true if this relocation is R_*_GLOB_DAT type. Return false
+ /// otherwise.
+ bool isGlobDat() const { return isGlobDat(Type); }
+
/// Return true if this relocation is R_*_IRELATIVE type. Return false
/// otherwise.
bool isIRelative() const { return isIRelative(Type); }
diff --git a/bolt/include/bolt/Passes/PatchEntries.h b/bolt/include/bolt/Passes/PatchEntries.h
index fa6b5811a4c3b..aed3451d78f86 100644
--- a/bolt/include/bolt/Passes/PatchEntries.h
+++ b/bolt/include/bolt/Passes/PatchEntries.h
@@ -33,6 +33,12 @@ class PatchEntries : public BinaryFunctionPass {
public:
explicit PatchEntries() : BinaryFunctionPass(false) {}
+ static size_t getPatchSize(const BinaryContext &BC) {
+ InstructionListType Seq;
+ BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
+ return BC.computeCodeSize(Seq.begin(), Seq.end());
+ }
+
const char *getName() const override { return "patch-entries"; }
Error runOnFunctions(BinaryContext &BC) override;
};
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 92a2763e28c4d..3934c8b6ed79e 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -473,6 +473,9 @@ class RewriteInstance {
/// Track next available address for new allocatable sections.
uint64_t NextAvailableAddress{0};
+ uint64_t BOLTReservedStartAddress{0};
+ uint64_t BOLTReservedEndAddress{0};
+
/// Location and size of dynamic relocations.
std::optional<uint64_t> DynamicRelocationsAddress;
uint64_t DynamicRelocationsSize{0};
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 377de5a994a11..5b382d318dffb 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -319,7 +319,7 @@ bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {
(SymbolName == "__hot_data_start" || SymbolName == "__hot_data_end"))
return true;
- if (SymbolName == "_end")
+ if (SymbolName == "_end" && !IsLinuxKernel)
return true;
return false;
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 50b8bd8f8e343..02a69ae296001 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -401,11 +401,11 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
assert((Function.empty() || !(*Function.begin()).isCold()) &&
"first basic block should never be cold");
- // Emit UD2 at the beginning if requested by user.
+ // Emit undefined instruction at the beginning if requested by user.
if (!opts::BreakFunctionNames.empty()) {
for (std::string &Name : opts::BreakFunctionNames) {
if (Function.hasNameRegex(Name)) {
- Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
+ Streamer.emitBytes(BC.MIB->getUndefFillValue());
break;
}
}
diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp
index b16e0a4333aa2..c4f5e171657f0 100644
--- a/bolt/lib/Core/BinarySection.cpp
+++ b/bolt/lib/Core/BinarySection.cpp
@@ -130,6 +130,18 @@ void BinarySection::emitAsData(MCStreamer &Streamer,
}
#endif
+ if (!BC.isRISCV() && std::distance(ROI, ROE) > 1) {
+ errs() << "BOLT-WARNING: multiple relocations at the same offset:\n";
+ for (const auto &Relocation : make_range(ROI, ROE)) {
+ errs() << " "
+ << (Relocation.Symbol ? Relocation.Symbol->getName()
+ : StringRef("<none>"))
+ << " at offset 0x" << Twine::utohexstr(Relocation.Offset)
+ << " with type " << Relocation.Type << '\n';
+ }
+ ROI = std::prev(ROE);
+ }
+
size_t RelocationSize = Relocation::emit(ROI, ROE, &Streamer);
SectionOffset += RelocationSize;
}
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index e9a9741bc3716..203d3c2297dd8 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -900,6 +900,19 @@ bool Relocation::isRelative(uint64_t Type) {
}
}
+bool Relocation::isGlobDat(uint64_t Type) {
+ switch (Arch) {
+ default:
+ llvm_unreachable("Unsupported architecture");
+ case Triple::aarch64:
+ return Type == ELF::R_AARCH64_GLOB_DAT;
+ case Triple::riscv64:
+ return Type == ELF::R_RISCV_64;
+ case Triple::x86_64:
+ return Type == ELF::R_X86_64_GLOB_DAT;
+ }
+}
+
bool Relocation::isIRelative(uint64_t Type) {
switch (Arch) {
default:
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index f49761f229ba6..cd12e86454128 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -1217,21 +1217,19 @@ Error LinuxKernelRewriter::readExceptionTable(StringRef SectionName) {
<< ExceptionsSection->getSize() / ExceptionTableEntrySize
<< " exception table entries\n";
- return Error::success();
-}
-
-/// Depending on the value of CONFIG_BUILDTIME_TABLE_SORT, the kernel expects
-/// the exception table to be sorted. Hence we have to sort it after code
-/// reordering.
-Error LinuxKernelRewriter::rewriteExceptionTable() {
// Disable output of functions with exceptions before rewrite support is
// added.
for (BinaryFunction *BF : FunctionsWithExceptions)
- BF->setSimple(false);
+ BF->setIgnored();
return Error::success();
}
+/// Depending on the value of CONFIG_BUILDTIME_TABLE_SORT, the kernel expects
+/// the exception table to be sorted. Hence we have to sort it after code
+/// reordering.
+Error LinuxKernelRewriter::rewriteExceptionTable() { return Error::success(); }
+
/// .parainsrtuctions section contains information for patching parvirtual call
/// instructions during runtime. The entries in the section are in the form:
///
@@ -1297,6 +1295,10 @@ Error LinuxKernelRewriter::readParaInstructions() {
}
}
+ // Disable output of functions with paravirtual instructions before the
+ // rewrite support is complete.
+ skipFunctionsWithAnnotation("ParaSite");
+
BC.outs() << "BOLT-INFO: parsed " << EntryID << " paravirtual patch sites\n";
return Error::success();
@@ -1312,7 +1314,7 @@ void LinuxKernelRewriter::skipFunctionsWithAnnotation(
return BC.MIB->hasAnnotation(Inst, Annotation);
});
if (HasAnnotation) {
- BF.setSimple(false);
+ BF.setIgnored();
break;
}
}
@@ -1320,10 +1322,6 @@ void LinuxKernelRewriter::skipFunctionsWithAnnotation(
}
Error LinuxKernelRewriter::rewriteParaInstructions() {
- // Disable output of functions with paravirtual instructions before the
- // rewrite support is complete.
- skipFunctionsWithAnnotation("ParaSite");
-
return Error::success();
}
@@ -1658,7 +1656,7 @@ Error LinuxKernelRewriter::readPCIFixupTable() {
if (const uint64_t Offset = HookAddress - BF->getAddress()) {
BC.errs() << "BOLT-WARNING: PCI fixup detected in the middle of function "
<< *BF << " at offset 0x" << Twine::utohexstr(Offset) << '\n';
- BF->setSimple(false);
+ BF->setIgnored();
}
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index e408fa5065a90..3174e986517fd 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1046,7 +1046,13 @@ void RewriteInstance::discoverFileObjects() {
continue;
}
- if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) {
+ if (SymName == getBOLTReservedStart()) {
+ BOLTReservedStartAddress = SymbolAddress;
+ registerName(SymbolSize);
+ continue;
+ }
+ if (SymName == getBOLTReservedEnd()) {
+ BOLTReservedEndAddress = SymbolAddress;
registerName(SymbolSize);
continue;
}
@@ -1316,7 +1322,8 @@ void RewriteInstance::discoverFileObjects() {
/*CheckPastEnd*/ false,
/*UseMaxSize*/ true);
if (BF) {
- assert(Rel.isRelative() && "Expected relative relocation for island");
+ assert((Rel.isRelative() || Rel.isGlobDat()) &&
+ "Unexpected relocation for island");
BC->logBOLTErrorsAndQuitOnFatal(
BF->markIslandDynamicRelocationAtAddress(RelAddress));
}
@@ -1324,10 +1331,8 @@ void RewriteInstance::discoverFileObjects() {
}
}
- if (!BC->IsLinuxKernel) {
- // Read all relocations now that we have binary functions mapped.
- processRelocations();
- }
+ // Read all relocations now that we have binary functions mapped.
+ processRelocations();
registerFragments();
FileSymbols.clear();
@@ -1337,30 +1342,36 @@ void RewriteInstance::discoverFileObjects() {
}
void RewriteInstance::discoverBOLTReserved() {
- BinaryData *StartBD = BC->getBinaryDataByName(getBOLTReservedStart());
- BinaryData *EndBD = BC->getBinaryDataByName(getBOLTReservedEnd());
- if (!StartBD != !EndBD) {
+ if (!BOLTReservedStartAddress != !BOLTReservedEndAddress) {
BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: "
<< getBOLTReservedStart() << ", " << getBOLTReservedEnd()
<< '\n';
exit(1);
}
- if (!StartBD)
- return;
-
- if (StartBD->getAddress() >= EndBD->getAddress()) {
- BC->errs() << "BOLT-ERROR: invalid reserved space boundaries\n";
+ if (BC->IsLinuxKernel && BC->HasRelocations && !BOLTReservedStartAddress) {
+ BC->errs() << "BOLT-ERROR: BOLT for Linux in relocation mode requires BOLT "
+ "reserved space\n";
exit(1);
}
- BC->BOLTReserved = AddressRange(StartBD->getAddress(), EndBD->getAddress());
- BC->outs() << "BOLT-INFO: using reserved space for allocating new sections\n";
- PHDRTableOffset = 0;
- PHDRTableAddress = 0;
- NewTextSegmentAddress = 0;
- NewTextSegmentOffset = 0;
- NextAvailableAddress = BC->BOLTReserved.start();
+ if (BOLTReservedStartAddress) {
+ if (BOLTReservedStartAddress >= BOLTReservedEndAddress) {
+ BC->errs() << "BOLT-ERROR: invalid reserved space boundaries\n";
+ exit(1);
+ }
+
+ BC->BOLTReserved =
+ AddressRange(BOLTReservedStartAddress, BOLTReservedEndAddress);
+ BC->outs()
+ << "BOLT-INFO: using reserved space for allocating new sections\n";
+
+ PHDRTableOffset = 0;
+ PHDRTableAddress = 0;
+ NewTextSegmentAddress = 0;
+ NewTextSegmentOffset = 0;
+ NextAvailableAddress = BC->BOLTReserved.start();
+ }
}
Error RewriteInstance::discoverRtFiniAddress() {
@@ -1813,6 +1824,10 @@ void RewriteInstance::adjustFunctionBoundaries() {
BFE = BC->getBinaryFunctions().end();
BFI != BFE; ++BFI) {
BinaryFunction &Function = BFI->second;
+
+ if (Function.getAddress() == BOLTReservedStartAddress)
+ continue;
+
const BinaryFunction *NextFunction = nullptr;
if (std::next(BFI) != BFE)
NextFunction = &std::next(BFI)->second;
@@ -2004,11 +2019,6 @@ Error RewriteInstance::readSpecialSections() {
BC->HasRelocations =
HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE);
- if (BC->IsLinuxKernel && BC->HasRelocations) {
- BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";
- BC->HasRelocations = false;
- }
-
BC->IsStripped = !HasSymbolTable;
if (BC->IsStripped && !opts::AllowStripped) {
@@ -2263,6 +2273,17 @@ bool RewriteInstance::analyzeRelocation(
SymbolAddress = BD ? BD->getAddress() : 0;
}
}
+
+ if (BC->IsLinuxKernel) {
+ if (BC->isX86()) {
+ if (StringSwitch<bool>(SymbolName)
+ .Cases(".data..percpu", "fixed_percpu_data", true)
+ .Default(false) ||
+ SymbolName.find("__per_cpu_") != std::string::npos)
+ return true;
+ }
+ }
+
// For PIE or dynamic libs, the linker may choose not to put the relocation
// result at the address if it is a X86_64_64 one because it will emit a
// dynamic relocation (X86_RELATIVE) for the dynamic linker and loader to
@@ -2561,6 +2582,21 @@ void RewriteInstance::readRelocations(const SectionRef &Section) {
SectionRef RelocatedSection = *SecIter;
StringRef RelocatedSectionName = cantFail(RelocatedSection.getName());
+
+ if (BC->IsLinuxKernel) {
+ if (BC->isX86()) {
+ if (StringSwitch<bool>(RelocatedSectionName)
+ .Case(".data..percpu", true)
+ .Default(false))
+ return;
+ }
+ if (StringSwitch<bool>(RelocatedSectionName)
+ .Cases("__ksymtab", "__ksymtab_gpl", "__bug_table",
+ ".altinstructions", true)
+ .Default(false))
+ return;
+ }
+
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is "
<< RelocatedSectionName << '\n');
@@ -2594,6 +2630,19 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
if (Relocation::skipRelocationType(RType))
return;
+ if (BC->IsLinuxKernel) {
+ BinaryData *BDStartJumpTable =
+ BC->getBinaryDataByName("__start___jump_table");
+ BinaryData *BDStopJumpTable =
+ BC->getBinaryDataByName("__stop___jump_table");
+ if (BDStartJumpTable && BDStopJumpTable) {
+ uint64_t JTStartAddress = BDStartJumpTable->getAddress();
+ uint64_t JTEndAddress = BDStopJumpTable->getAddress();
+ if (JTStartAddress <= Rel.getOffset() && Rel.getOffset() < JTEndAddress)
+ return;
+ }
+ }
+
// Adjust the relocation type as the linker might have skewed it.
if (BC->isX86() && (RType & ELF::R_X86_64_converted_reloc_bit)) {
if (opts::Verbosity >= 1)
>From b37c947c7e03fa21a0d5c26c7a8e76d7f7c86178 Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Wed, 12 Mar 2025 17:14:59 +0800
Subject: [PATCH 6/6] [BOLT][Linux] Add support for relocation mode
---
bolt/include/bolt/Core/BinaryFunction.h | 8 ++++
bolt/include/bolt/Passes/PatchEntries.h | 11 +++--
bolt/lib/Passes/PatchEntries.cpp | 38 ++++++----------
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 32 ++++++++++++++
bolt/lib/Rewrite/RewriteInstance.cpp | 43 +++++++++++++++----
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 4 ++
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 4 ++
bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 4 ++
8 files changed, 108 insertions(+), 36 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index ef43183aa9e4e..766aac652cdde 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -297,6 +297,9 @@ class BinaryFunction {
// True if address of this function can not be changed
bool KeepAddress{false};
+ // True if code of this function might be changed at run time
+ bool MayChange{false};
+
/// True if the original function code has all necessary relocations to track
/// addresses of functions emitted to new locations. Typically set for
/// functions that we are not going to emit.
@@ -1350,6 +1353,9 @@ class BinaryFunction {
/// Return true if address of this function can not be changed
bool mustKeepAddress() const { return KeepAddress; }
+ /// Return true if code of this function might be changed at run time
+ bool mayChange() const { return MayChange; }
+
/// Return true if the function contains explicit or implicit indirect branch
/// to its split fragments, e.g., split jump table, landing pad in split
/// fragment.
@@ -1761,6 +1767,8 @@ class BinaryFunction {
/// Mark function that should not be emitted.
void setIgnored();
+ void setMayChange() { MayChange = true; }
+
void setIsPatched(bool V) { IsPatched = V; }
void setHasIndirectTargetToSplitFragment(bool V) {
diff --git a/bolt/include/bolt/Passes/PatchEntries.h b/bolt/include/bolt/Passes/PatchEntries.h
index aed3451d78f86..e4982b5c6529c 100644
--- a/bolt/include/bolt/Passes/PatchEntries.h
+++ b/bolt/include/bolt/Passes/PatchEntries.h
@@ -33,10 +33,15 @@ class PatchEntries : public BinaryFunctionPass {
public:
explicit PatchEntries() : BinaryFunctionPass(false) {}
+ // Calculate the size of the patch.
static size_t getPatchSize(const BinaryContext &BC) {
- InstructionListType Seq;
- BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
- return BC.computeCodeSize(Seq.begin(), Seq.end());
+ static size_t PatchSize = 0;
+ if (!PatchSize) {
+ InstructionListType Seq;
+ BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
+ PatchSize = BC.computeCodeSize(Seq.begin(), Seq.end());
+ }
+ return PatchSize;
}
const char *getName() const override { return "patch-entries"; }
diff --git a/bolt/lib/Passes/PatchEntries.cpp b/bolt/lib/Passes/PatchEntries.cpp
index 981d1b70af907..e75fc323e7cf7 100644
--- a/bolt/lib/Passes/PatchEntries.cpp
+++ b/bolt/lib/Passes/PatchEntries.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "bolt/Passes/PatchEntries.h"
+#include "bolt/Utils/CommandLineOpts.h"
#include "bolt/Utils/NameResolver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/CommandLine.h"
@@ -32,29 +33,10 @@ namespace llvm {
namespace bolt {
Error PatchEntries::runOnFunctions(BinaryContext &BC) {
- if (!opts::ForcePatch) {
- // Mark the binary for patching if we did not create external references
- // for original code in any of functions we are not going to emit.
- bool NeedsPatching = llvm::any_of(
- llvm::make_second_range(BC.getBinaryFunctions()),
- [&](BinaryFunction &BF) {
- return !BC.shouldEmit(BF) && !BF.hasExternalRefRelocations();
- });
-
- if (!NeedsPatching)
- return Error::success();
- }
-
- if (opts::Verbosity >= 1)
- BC.outs() << "BOLT-INFO: patching entries in original code\n";
+ BC.outs() << "BOLT-INFO: patching entries in original code\n";
- // Calculate the size of the patch.
- static size_t PatchSize = 0;
- if (!PatchSize) {
- InstructionListType Seq;
- BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
- PatchSize = BC.computeCodeSize(Seq.begin(), Seq.end());
- }
+ static size_t PatchSize = getPatchSize(BC);
+ BC.outs() << "BOLT-INFO: patch size is " << PatchSize << "\n";
for (auto &BFI : BC.getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
@@ -63,8 +45,16 @@ Error PatchEntries::runOnFunctions(BinaryContext &BC) {
if (!BC.shouldEmit(Function))
continue;
+ bool MustPatch = opts::ForcePatch;
+
+ // In relocation mode, a copy will be created and only the copy can be
+ // changed. To avoid undefined behaviors, we must make the original function
+ // jump to the copy.
+ if (BC.HasRelocations && Function.mayChange())
+ MustPatch = true;
+
// Check if we can skip patching the function.
- if (!opts::ForcePatch && !Function.hasEHRanges() &&
+ if (!MustPatch && !Function.hasEHRanges() &&
Function.getSize() < PatchThreshold)
continue;
@@ -100,7 +90,7 @@ Error PatchEntries::runOnFunctions(BinaryContext &BC) {
if (!Success) {
// We can't change output layout for AArch64 due to LongJmp pass
if (BC.isAArch64()) {
- if (opts::ForcePatch) {
+ if (MustPatch) {
BC.errs() << "BOLT-ERROR: unable to patch entries in " << Function
<< "\n";
return createFatalBOLTError("");
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index cd12e86454128..9e99681c6418b 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Passes/PatchEntries.h"
#include "bolt/Rewrite/MetadataRewriter.h"
#include "bolt/Rewrite/MetadataRewriters.h"
#include "bolt/Utils/CommandLineOpts.h"
@@ -218,6 +219,8 @@ class LinuxKernelRewriter final : public MetadataRewriter {
bool InitValue{false};
bool Nop{false};
MCSymbol *JumpInstLabel{nullptr};
+ BinarySection *Sec{nullptr};
+ uint64_t JumpAddress{0};
BinaryFunction *BF{nullptr};
};
std::vector<JumpInfoEntry> JumpInfo;
@@ -1754,6 +1757,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
JumpInfo.push_back(JumpInfoEntry());
JumpInfoEntry &Info = JumpInfo.back();
Info.Likely = KeyAddress & 1;
+ Info.JumpAddress = JumpAddress;
if (opts::DumpStaticKeys) {
BC.outs() << "Static key jump entry: " << EntryID
@@ -1776,6 +1780,9 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
assert(BF->getOriginSection() &&
"the function did not originate from the file");
Info.BF = BF;
+ Info.Sec = BF->getOriginSection();
+
+ BF->setMayChange();
MCInst *Inst = BF->getInstructionAtOffset(JumpAddress - BF->getAddress());
if (!Inst)
@@ -2028,6 +2035,31 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
llvm_unreachable("Unsupported architecture");
}
+ if (BC.HasRelocations) {
+ // To avoid undefined behaviors, fill the jump address with Undef
+
+ size_t PatchSize = PatchEntries::getPatchSize(BC);
+ assert(BF->isPatched());
+ assert(Info.JumpAddress != JumpAddress);
+
+ bool NotOverlap =
+ BF->forEachEntryPoint([&](uint64_t EntryOffset, const MCSymbol *) {
+ uint64_t EntryAddress = EntryOffset + BF->getAddress();
+ return Info.JumpAddress >= EntryAddress + PatchSize ||
+ Info.JumpAddress + Size <= EntryAddress;
+ });
+
+ if (NotOverlap)
+ Info.Sec->addPatch(Info.JumpAddress - Info.Sec->getAddress(),
+ BC.MIB->getUndefFillValue());
+ else
+ BC.errs()
+ << "BOLT-WARNING: Skip writing an undefined instruction at static "
+ "key jump address 0x"
+ << Twine::utohexstr(Info.JumpAddress)
+ << " since that address is overlapping an entry point patch\n";
+ }
+
// Check if we need to convert jump instruction into a nop.
if (!Info.Nop)
continue;
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 3174e986517fd..743bd1e728f61 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -2643,6 +2643,12 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
}
}
+ if (BC->IsLinuxKernel) {
+ if (BC->isInRange("__start___jump_table", "__stop___jump_table",
+ Rel.getOffset()))
+ return;
+ }
+
// Adjust the relocation type as the linker might have skewed it.
if (BC->isX86() && (RType & ELF::R_X86_64_converted_reloc_bit)) {
if (opts::Verbosity >= 1)
@@ -2719,8 +2725,8 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
MCSymbol *ReferencedSymbol = nullptr;
if (!IsSectionRelocation) {
- if (BinaryData *BD = BC->getBinaryDataByName(SymbolName))
- ReferencedSymbol = BD->getSymbol();
+ if (BC->getBinaryDataByName(SymbolName))
+ ReferencedSymbol = BC->Ctx->getOrCreateSymbol(SymbolName);
else if (BC->isGOTSymbol(SymbolName))
if (BinaryData *BD = BC->getGOTSymbol())
ReferencedSymbol = BD->getSymbol();
@@ -2860,7 +2866,8 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
Addend = Address;
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: forcing relocation against symbol "
<< SymbolName << " with addend " << Addend << '\n');
- } else if (ReferencedBF) {
+ } else if (ReferencedBF && ReferencedSection &&
+ *ReferencedBF->getOriginSection() == *ReferencedSection) {
ReferencedSymbol = ReferencedBF->getSymbol();
uint64_t RefFunctionOffset = 0;
@@ -2909,7 +2916,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
dbgs() << formatv(" at offset {0:x}", RefFunctionOffset);
dbgs() << '\n';
});
- } else {
+ } else if (!ReferencedBF) {
if (IsToCode && SymbolAddress) {
// This can happen e.g. with PIC-style jump tables.
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: no corresponding function for "
@@ -4119,9 +4126,7 @@ void RewriteInstance::mapAllocatableSections(
}
} else if (SType == ST_READWRITE) {
NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
- // Restore NextAvailableAddress if no new writable sections
- if (!NewWritableSegmentSize)
- NextAvailableAddress = LastNextAvailableAddress;
+ // Even empty sections should be kept for their page align effects
}
}
}
@@ -4538,9 +4543,11 @@ RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
addSection(NewSection, Section);
}
- // Sort all allocatable sections by their offset.
+ // Sort all allocatable sections by their offset and size, to avoid that a
+ // zero size section cause a preceding non-zero size section truncated.
llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) {
- return A.second.sh_offset < B.second.sh_offset;
+ return std::make_tuple(A.second.sh_offset, A.second.sh_size) <
+ std::make_tuple(B.second.sh_offset, B.second.sh_size);
});
// Fix section sizes to prevent overlapping.
@@ -4698,6 +4705,10 @@ void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {
NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();
else
NewEhdr.e_entry = getNewFunctionAddress(NewEhdr.e_entry);
+
+ if (BC->IsLinuxKernel)
+ NewEhdr.e_entry = Obj.getHeader().e_entry;
+
assert((NewEhdr.e_entry || !Obj.getHeader().e_entry) &&
"cannot find new address for entry point");
}
@@ -4949,10 +4960,24 @@ void RewriteInstance::updateELFSymbolTable(
goto registerSymbol;
}
+ if (SymbolName->starts_with("__bolt_reserved_")) {
+ NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
+ goto registerSymbol;
+ }
+
if (Function) {
// If the symbol matched a function that was not emitted, update the
// corresponding section index but otherwise leave it unchanged.
if (Function->isEmitted()) {
+ if (BC->HasRelocations && !Function->IsPatched) {
+ ELFSymTy OrgSymbol = Symbol;
+ SmallVector<char, 256> Buf;
+ OrgSymbol.st_name =
+ AddToStrTab(Twine(*SymbolName).concat(".org.0").toStringRef(Buf));
+ OrgSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
+ if (!IsDynSym)
+ Symbols.emplace_back(OrgSymbol);
+ }
NewSymbol.st_value = Function->getOutputAddress();
NewSymbol.st_size = Function->getOutputSize();
NewSymbol.st_shndx = Function->getCodeSection()->getIndex();
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index ae5df0c02f056..02843b6857e00 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -1958,6 +1958,10 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
return StringRef("\0\0\0\0", 4);
}
+ StringRef getUndefFillValue() const override {
+ return StringRef("\xff\xff\x00\x00", 4); // UDF
+ }
+
void createReturn(MCInst &Inst) const override {
Inst.setOpcode(AArch64::RET);
Inst.clear();
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index f8c83b09395f5..5330506659291 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -238,6 +238,10 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
return StringRef("\0\0\0\0", 4);
}
+ StringRef getUndefFillValue() const override {
+ return StringRef("\x73\x10\x00\xc0", 4); // UNIMP
+ }
+
void createCall(unsigned Opcode, MCInst &Inst, const MCSymbol *Target,
MCContext *Ctx) {
Inst.setOpcode(Opcode);
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index 465533ee71f2b..a12aa39adc312 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -413,6 +413,10 @@ class X86MCPlusBuilder : public MCPlusBuilder {
StringRef getTrapFillValue() const override { return StringRef("\314", 1); }
+ StringRef getUndefFillValue() const override {
+ return StringRef("\x0f\x0b", 2); // UD2
+ }
+
struct IndJmpMatcherFrag1 : MCInstMatcher {
std::unique_ptr<MCInstMatcher> Base;
std::unique_ptr<MCInstMatcher> Scale;
More information about the llvm-commits
mailing list