[llvm] [BOLT][Linux] Refactor to support different architectures and Linux versions (PR #130398)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 7 22:17:23 PST 2025
https://github.com/FLZ101 created https://github.com/llvm/llvm-project/pull/130398
* utilize LinuxKernelVersion to simplify parsing of alternative instruction entries and exception table entries
* refactor static keys handling to make it work with relocation mode and function splitting, and also reduce duplicated code
>From 85b51bda7eda5a518f585820b3cc99cd8cf3680b Mon Sep 17 00:00:00 2001
From: Franklin <zhangfenglei at huawei.com>
Date: Sat, 8 Mar 2025 14:05:24 +0800
Subject: [PATCH] [BOLT][Linux] Refactor to support different architectures and
Linux versions
* utilize LinuxKernelVersion to simplify parsing of alternative
instruction entries and exception table entries
* refactor static keys handling to make it work with relocation
mode and function splitting, and also reduce duplicated code
---
bolt/docs/CommandLineArgumentReference.md | 12 -
bolt/include/bolt/Core/BinaryContext.h | 13 +-
bolt/include/bolt/Core/BinaryFunction.h | 11 +
bolt/include/bolt/Core/FunctionLayout.h | 4 +
bolt/include/bolt/Core/Linker.h | 7 +
bolt/include/bolt/Core/MCPlusBuilder.h | 3 +-
bolt/include/bolt/Rewrite/MetadataRewriter.h | 8 +-
bolt/include/bolt/Rewrite/MetadataRewriters.h | 8 +-
bolt/include/bolt/Rewrite/RewriteInstance.h | 2 +
bolt/lib/Core/BinaryContext.cpp | 17 +
bolt/lib/Core/JumpTable.cpp | 2 +-
bolt/lib/Core/MCPlusBuilder.cpp | 9 +-
bolt/lib/Rewrite/BuildIDRewriter.cpp | 8 +-
bolt/lib/Rewrite/CMakeLists.txt | 1 +
bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 411 +++++++++---------
bolt/lib/Rewrite/MetadataRewriter.cpp | 20 +
bolt/lib/Rewrite/PseudoProbeRewriter.cpp | 8 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 19 +-
bolt/lib/Rewrite/SDTRewriter.cpp | 7 +-
bolt/test/X86/linux-alt-instruction.s | 38 +-
20 files changed, 324 insertions(+), 284 deletions(-)
create mode 100644 bolt/lib/Rewrite/MetadataRewriter.cpp
diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index f3881c9a640a9..2c880388e3faa 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -56,14 +56,6 @@
Allow processing of stripped binaries
-- `--alt-inst-feature-size=<uint>`
-
- Size of feature field in .altinstructions
-
-- `--alt-inst-has-padlen`
-
- Specify that .altinstructions has padlen field
-
- `--asm-dump[=<dump folder>]`
Dump function into assembly
@@ -254,10 +246,6 @@
Redirect journaling to a file instead of stdout/stderr
-- `--long-jump-labels`
-
- Always use long jumps/nops for Linux kernel static keys
-
- `--match-profile-with-function-hash`
Match profile with function hash
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 8bec1db70e25a..77104d4c2f9cc 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -911,7 +911,11 @@ class BinaryContext {
/// Return a value of the global \p Symbol or an error if the value
/// was not set.
ErrorOr<uint64_t> getSymbolValue(const MCSymbol &Symbol) const {
- const BinaryData *BD = getBinaryDataByName(Symbol.getName());
+ return getSymbolValue(Symbol.getName());
+ }
+
+ ErrorOr<uint64_t> getSymbolValue(StringRef Name) const {
+ const BinaryData *BD = getBinaryDataByName(Name);
if (!BD)
return std::make_error_code(std::errc::bad_address);
return BD->getAddress();
@@ -1237,6 +1241,13 @@ class BinaryContext {
return const_cast<BinaryContext *>(this)->getSectionForAddress(Address);
}
+ ErrorOr<BinarySection &> getSectionForOutputAddress(uint64_t Address);
+ ErrorOr<const BinarySection &>
+ getSectionForOutputAddress(uint64_t Address) const {
+ return const_cast<BinaryContext *>(this)->getSectionForOutputAddress(
+ Address);
+ }
+
/// Return internal section representation for a section in a file.
BinarySection *getSectionForSectionRef(SectionRef Section) const {
return SectionRefToBinarySection.lookup(Section);
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 942840a7621fd..20caebe4b129c 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1243,6 +1243,17 @@ class BinaryFunction {
return Islands->FunctionColdConstantIslandLabel;
}
+ const FunctionFragment *
+ getFunctionFragmentForOutputAddress(uint64_t OutputAddress) const {
+ for (const FunctionFragment &FF : Layout.fragments()) {
+ uint64_t Address = FF.getAddress();
+ uint64_t Size = FF.getImageSize();
+ if (Address <= OutputAddress && OutputAddress < Address + Size)
+ return &FF;
+ }
+ return nullptr;
+ }
+
/// Return true if this is a function representing a PLT entry.
bool isPLTFunction() const { return PLTSymbol != nullptr; }
diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h
index ee4dd689b8dd6..65b80051862c4 100644
--- a/bolt/include/bolt/Core/FunctionLayout.h
+++ b/bolt/include/bolt/Core/FunctionLayout.h
@@ -117,6 +117,10 @@ class FunctionFragment {
uint64_t getFileOffset() const { return FileOffset; }
void setFileOffset(uint64_t Offset) { FileOffset = Offset; }
+ uint8_t *getOutputData() const {
+ return reinterpret_cast<uint8_t *>(getImageAddress());
+ }
+
unsigned size() const { return Size; };
bool empty() const { return size() == 0; };
iterator begin();
diff --git a/bolt/include/bolt/Core/Linker.h b/bolt/include/bolt/Core/Linker.h
index 66b3ad18e3c7b..1e0876a0e13d9 100644
--- a/bolt/include/bolt/Core/Linker.h
+++ b/bolt/include/bolt/Core/Linker.h
@@ -46,6 +46,13 @@ class BOLTLinker {
/// Return the address and size of a symbol or std::nullopt if it cannot be
/// found.
virtual std::optional<SymbolInfo> lookupSymbolInfo(StringRef Name) const = 0;
+
+ /// Return the address of a symbol or std::nullopt if it cannot be found.
+ std::optional<uint64_t> lookupSymbol(StringRef Name) const {
+ if (const auto Info = lookupSymbolInfo(Name))
+ return Info->Address;
+ return std::nullopt;
+ }
};
} // namespace bolt
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index fbb853656fb91..52643ffcd5b78 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1211,8 +1211,9 @@ class MCPlusBuilder {
/// Set the label of \p Inst or return the existing label for the instruction.
/// This label will be emitted right before \p Inst is emitted to MCStreamer.
+ /// If \p Temp is true, then this label does not survive in the symbol table.
MCSymbol *getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
- MCContext *Ctx) const;
+ MCContext *Ctx, bool Temp = true) const;
/// Set the label of \p Inst. This label will be emitted right before \p Inst
/// is emitted to MCStreamer.
diff --git a/bolt/include/bolt/Rewrite/MetadataRewriter.h b/bolt/include/bolt/Rewrite/MetadataRewriter.h
index 6ff8f0af7a8e6..6988e5de4e6bd 100644
--- a/bolt/include/bolt/Rewrite/MetadataRewriter.h
+++ b/bolt/include/bolt/Rewrite/MetadataRewriter.h
@@ -19,6 +19,8 @@
namespace llvm {
namespace bolt {
+class RewriteInstance;
+
/// Base class for handling file sections with metadata. In this context,
/// metadata encompasses a wide range of data that references code and other
/// data. Such metadata may or may not have an impact on program execution.
@@ -34,10 +36,14 @@ class MetadataRewriter {
StringRef Name;
protected:
+ RewriteInstance &RI;
+
/// Provides access to the binary context.
BinaryContext &BC;
- MetadataRewriter(StringRef Name, BinaryContext &BC) : Name(Name), BC(BC) {}
+ MetadataRewriter(StringRef Name, RewriteInstance &RI);
+
+ std::optional<uint64_t> lookupSymbol(const StringRef Name);
public:
virtual ~MetadataRewriter() = default;
diff --git a/bolt/include/bolt/Rewrite/MetadataRewriters.h b/bolt/include/bolt/Rewrite/MetadataRewriters.h
index b71bd6cad2505..76face9888235 100644
--- a/bolt/include/bolt/Rewrite/MetadataRewriters.h
+++ b/bolt/include/bolt/Rewrite/MetadataRewriters.h
@@ -19,13 +19,13 @@ class BinaryContext;
// The list of rewriter build functions.
-std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createLinuxKernelRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createBuildIDRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createBuildIDRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createPseudoProbeRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createPseudoProbeRewriter(RewriteInstance &);
-std::unique_ptr<MetadataRewriter> createSDTRewriter(BinaryContext &);
+std::unique_ptr<MetadataRewriter> createSDTRewriter(RewriteInstance &);
} // namespace bolt
} // namespace llvm
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 42094cb732107..92a2763e28c4d 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -42,6 +42,8 @@ class ProfileReaderBase;
/// optimizations) and rewriting. It also has the logic to coordinate such
/// events.
class RewriteInstance {
+ friend class MetadataRewriter;
+
public:
// This constructor has complex initialization that can fail during
// construction. Constructors can’t return errors, so clients must test \p Err
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index f9fc536f3569a..377de5a994a11 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2110,6 +2110,23 @@ ErrorOr<BinarySection &> BinaryContext::getSectionForAddress(uint64_t Address) {
return std::make_error_code(std::errc::bad_address);
}
+ErrorOr<BinarySection &>
+BinaryContext::getSectionForOutputAddress(uint64_t Address) {
+ for (auto &Sec : allocatableSections()) {
+ // Skip pseudo sections that serve a purpose of creating a corresponding
+ // entry in section header table
+ if (Sec.getOutputContents().empty())
+ continue;
+
+ uint64_t OutputAddress = Sec.getOutputAddress();
+ uint64_t OutputSize = Sec.getOutputSize();
+ if (OutputAddress && OutputAddress <= Address &&
+ Address < OutputAddress + OutputSize)
+ return Sec;
+ }
+ return std::make_error_code(std::errc::bad_address);
+}
+
ErrorOr<StringRef>
BinaryContext::getSectionNameForAddress(uint64_t Address) const {
if (ErrorOr<const BinarySection &> Section = getSectionForAddress(Address))
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 65e1032c579b5..d3ca951d7e453 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -85,7 +85,7 @@ void bolt::JumpTable::updateOriginal() {
uint64_t EntryOffset = BaseOffset;
for (MCSymbol *Entry : Entries) {
const uint64_t RelType =
- Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+ Type == JTT_NORMAL ? Relocation::getAbs64() : Relocation::getPC32();
const uint64_t RelAddend =
Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
// Replace existing relocation with the new one to allow any modifications
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index 7ff7a2288451c..b0aa40f9eac16 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -288,12 +288,17 @@ MCSymbol *MCPlusBuilder::getInstLabel(const MCInst &Inst) const {
}
MCSymbol *MCPlusBuilder::getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
- MCContext *Ctx) const {
+ MCContext *Ctx, bool Temp) const {
MCSymbol *Label = getInstLabel(Inst);
if (Label)
return Label;
- Label = Ctx->createNamedTempSymbol(Name);
+ if (Temp) {
+ Label = Ctx->createNamedTempSymbol(Name);
+ } else {
+ SmallVector<char, 16> Buf;
+ Label = Ctx->createLocalSymbol(Name.toStringRef(Buf));
+ }
setAnnotationOpValue(Inst, MCAnnotation::kLabel,
reinterpret_cast<int64_t>(Label));
return Label;
diff --git a/bolt/lib/Rewrite/BuildIDRewriter.cpp b/bolt/lib/Rewrite/BuildIDRewriter.cpp
index 83d0c9bfe182a..8a9c32619f6a9 100644
--- a/bolt/lib/Rewrite/BuildIDRewriter.cpp
+++ b/bolt/lib/Rewrite/BuildIDRewriter.cpp
@@ -39,8 +39,8 @@ class BuildIDRewriter final : public MetadataRewriter {
std::optional<uint64_t> BuildIDSize;
public:
- BuildIDRewriter(StringRef Name, BinaryContext &BC)
- : MetadataRewriter(Name, BC) {}
+ BuildIDRewriter(StringRef Name, RewriteInstance &RI)
+ : MetadataRewriter(Name, RI) {}
Error sectionInitializer() override;
@@ -108,6 +108,6 @@ Error BuildIDRewriter::postEmitFinalizer() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createBuildIDRewriter(BinaryContext &BC) {
- return std::make_unique<BuildIDRewriter>("build-id-rewriter", BC);
+llvm::bolt::createBuildIDRewriter(RewriteInstance &RI) {
+ return std::make_unique<BuildIDRewriter>("build-id-rewriter", RI);
}
diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index c83cf36982167..c403b8fcc33b2 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_library(LLVMBOLTRewrite
LinuxKernelRewriter.cpp
MachORewriteInstance.cpp
MetadataManager.cpp
+ MetadataRewriter.cpp
BuildIDRewriter.cpp
PseudoProbeRewriter.cpp
RewriteInstance.cpp
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 5a5e044184d0b..578ab40ec04ce 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -31,16 +31,6 @@ using namespace bolt;
namespace opts {
-static cl::opt<bool>
- AltInstHasPadLen("alt-inst-has-padlen",
- cl::desc("specify that .altinstructions has padlen field"),
- cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-
-static cl::opt<uint32_t>
- AltInstFeatureSize("alt-inst-feature-size",
- cl::desc("size of feature field in .altinstructions"),
- cl::init(2), cl::Hidden, cl::cat(BoltCategory));
-
static cl::opt<bool>
DumpAltInstructions("dump-alt-instructions",
cl::desc("dump Linux alternative instructions info"),
@@ -79,11 +69,6 @@ static cl::opt<bool>
cl::desc("dump Linux kernel static keys jump table"),
cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-static cl::opt<bool> LongJumpLabels(
- "long-jump-labels",
- cl::desc("always use long jumps/nops for Linux kernel static keys"),
- cl::init(false), cl::Hidden, cl::cat(BoltCategory));
-
static cl::opt<bool>
PrintORC("print-orc",
cl::desc("print ORC unwind information for instructions"),
@@ -94,7 +79,7 @@ static cl::opt<bool>
/// Linux kernel version
struct LKVersion {
LKVersion() {}
- LKVersion(unsigned Major, unsigned Minor, unsigned Rev)
+ LKVersion(unsigned Major, unsigned Minor, unsigned Rev = 0)
: Major(Major), Minor(Minor), Rev(Rev) {}
bool operator<(const LKVersion &Other) const {
@@ -229,13 +214,16 @@ class LinuxKernelRewriter final : public MetadataRewriter {
static constexpr size_t STATIC_KEYS_JUMP_ENTRY_SIZE = 8;
struct JumpInfoEntry {
- bool Likely;
- bool InitValue;
+ bool Likely{false};
+ bool InitValue{false};
+ bool Nop{false};
+ MCSymbol *JumpInstLabel{nullptr};
+ BinaryFunction *BF{nullptr};
};
- SmallVector<JumpInfoEntry, 16> JumpInfo;
+ std::vector<JumpInfoEntry> JumpInfo;
- /// Static key entries that need nop conversion.
- DenseSet<uint32_t> NopIDs;
+ // Use long jumps/nops for Linux kernel static keys
+ bool LongJumpLabels{false};
/// Section containing static call table.
ErrorOr<BinarySection &> StaticCallSection = std::errc::bad_address;
@@ -249,11 +237,6 @@ class LinuxKernelRewriter final : public MetadataRewriter {
};
using StaticCallListType = std::vector<StaticCallInfo>;
StaticCallListType StaticCallEntries;
-
- /// Section containing the Linux exception table.
- ErrorOr<BinarySection &> ExceptionsSection = std::errc::bad_address;
- static constexpr size_t EXCEPTION_TABLE_ENTRY_SIZE = 12;
-
/// Functions with exception handling code.
DenseSet<BinaryFunction *> FunctionsWithExceptions;
@@ -266,6 +249,15 @@ class LinuxKernelRewriter final : public MetadataRewriter {
/// .altinstructions section.
ErrorOr<BinarySection &> AltInstrSection = std::errc::bad_address;
+ struct AltInstrEntry {
+ uint64_t Offset{0};
+ uint64_t OrgInstrAddr{0};
+ uint64_t AltInstrAddr{0};
+ uint8_t Instrlen{0};
+ uint8_t Replacementlen{0};
+ };
+ std::vector<AltInstrEntry> AltInstrEntries;
+
/// Section containing Linux bug table.
ErrorOr<BinarySection &> BugTableSection = std::errc::bad_address;
@@ -314,7 +306,7 @@ class LinuxKernelRewriter final : public MetadataRewriter {
Error readStaticCalls();
Error rewriteStaticCalls();
- Error readExceptionTable();
+ Error readExceptionTable(StringRef SectionName);
Error rewriteExceptionTable();
/// Paravirtual instruction patch sites.
@@ -332,8 +324,6 @@ class LinuxKernelRewriter final : public MetadataRewriter {
/// Handle alternative instruction info from .altinstructions.
Error readAltInstructions();
void processAltInstructionsPostCFG();
- Error tryReadAltInstructions(uint32_t AltInstFeatureSize,
- bool AltInstHasPadLen, bool ParseOnly);
/// Read .pci_fixup
Error readPCIFixupTable();
@@ -344,8 +334,8 @@ class LinuxKernelRewriter final : public MetadataRewriter {
Error updateStaticKeysJumpTablePostEmit();
public:
- LinuxKernelRewriter(BinaryContext &BC)
- : MetadataRewriter("linux-kernel-rewriter", BC) {}
+ LinuxKernelRewriter(RewriteInstance &RI)
+ : MetadataRewriter("linux-kernel-rewriter", RI) {}
Error preCFGInitializer() override {
if (Error E = detectLinuxKernelVersion())
@@ -359,7 +349,10 @@ class LinuxKernelRewriter final : public MetadataRewriter {
if (Error E = readStaticCalls())
return E;
- if (Error E = readExceptionTable())
+ if (Error E = readExceptionTable("__ex_table"))
+ return E;
+
+ if (Error E = readExceptionTable("__kvm_ex_table"))
return E;
if (Error E = readParaInstructions())
@@ -446,6 +439,8 @@ Error LinuxKernelRewriter::detectLinuxKernelVersion() {
LinuxKernelVersion = LKVersion(Major, Minor, Rev);
BC.outs() << "BOLT-INFO: Linux kernel version is " << Match[1].str()
<< "\n";
+ if (LinuxKernelVersion < LKVersion(5, 0))
+ return createStringError("Unsupported Linux kernel version");
return Error::success();
}
}
@@ -557,8 +552,8 @@ void LinuxKernelRewriter::processInstructionFixups() {
continue;
Fixup.Section.addRelocation(Fixup.Offset, &Fixup.Label,
- Fixup.IsPCRelative ? ELF::R_X86_64_PC32
- : ELF::R_X86_64_64,
+ Fixup.IsPCRelative ? Relocation::getPC32()
+ : Relocation::getAbs64(),
/*Addend*/ 0);
}
}
@@ -1074,7 +1069,7 @@ Error LinuxKernelRewriter::rewriteStaticCalls() {
StaticCallSection->getAddress() +
(Entry.ID - 1) * STATIC_CALL_ENTRY_SIZE;
StaticCallSection->addRelocation(EntryOffset, Entry.Label,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ Relocation::getPC32(), /*Addend*/ 0);
}
return Error::success();
@@ -1094,12 +1089,24 @@ Error LinuxKernelRewriter::rewriteStaticCalls() {
///
/// More info at:
/// https://www.kernel.org/doc/Documentation/x86/exception-tables.txt
-Error LinuxKernelRewriter::readExceptionTable() {
- ExceptionsSection = BC.getUniqueSectionByName("__ex_table");
+Error LinuxKernelRewriter::readExceptionTable(StringRef SectionName) {
+ ErrorOr<BinarySection &> ExceptionsSection =
+ BC.getUniqueSectionByName(SectionName);
if (!ExceptionsSection)
return Error::success();
- if (ExceptionsSection->getSize() % EXCEPTION_TABLE_ENTRY_SIZE)
+ size_t ExceptionTableEntrySize = 0;
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ ExceptionTableEntrySize = 12;
+ break;
+
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+ assert(ExceptionTableEntrySize && "exception table entry size is unknown");
+
+ if (ExceptionsSection->getSize() % ExceptionTableEntrySize)
return createStringError(errc::executable_format_error,
"exception table size error");
@@ -1111,7 +1118,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
while (Cursor && Cursor.tell() < ExceptionsSection->getSize()) {
const uint64_t InstAddress = AE.getPCRelAddress32(Cursor);
const uint64_t FixupAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t Data = AE.getU32(Cursor);
+ Cursor.seek(Cursor.tell() + ExceptionTableEntrySize - 8);
// Consume the status of the cursor.
if (!Cursor)
@@ -1125,8 +1132,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
if (opts::DumpExceptions) {
BC.outs() << "Exception Entry: " << EntryID << '\n';
BC.outs() << "\tInsn: 0x" << Twine::utohexstr(InstAddress) << '\n'
- << "\tFixup: 0x" << Twine::utohexstr(FixupAddress) << '\n'
- << "\tData: 0x" << Twine::utohexstr(Data) << '\n';
+ << "\tFixup: 0x" << Twine::utohexstr(FixupAddress) << '\n';
}
MCInst *Inst = nullptr;
@@ -1174,7 +1180,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
}
BC.outs() << "BOLT-INFO: parsed "
- << ExceptionsSection->getSize() / EXCEPTION_TABLE_ENTRY_SIZE
+ << ExceptionsSection->getSize() / ExceptionTableEntrySize
<< " exception table entries\n";
return Error::success();
@@ -1377,7 +1383,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
MCSymbol *Label =
BC.MIB->getOrCreateInstLabel(Inst, "__BUG_", BC.Ctx.get());
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, Label, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, Label,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1387,7 +1394,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
for (const uint32_t ID : FunctionBugList[&BF]) {
if (!EmittedIDs.count(ID)) {
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, nullptr, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, nullptr,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1399,95 +1407,69 @@ Error LinuxKernelRewriter::rewriteBugTable() {
/// The kernel can replace certain instruction sequences depending on hardware
/// it is running on and features specified during boot time. The information
/// about alternative instruction sequences is stored in .altinstructions
-/// section. The format of entries in this section is defined in
-/// arch/x86/include/asm/alternative.h:
-///
+/// section. The format of entries in this section is defined as
/// struct alt_instr {
/// s32 instr_offset;
/// s32 repl_offset;
-/// uXX feature;
+/// ...
/// u8 instrlen;
/// u8 replacementlen;
-/// u8 padlen; // present in older kernels
+/// ...
/// } __packed;
///
-/// Note that the structure is packed.
+/// Note that the structure is packed and field names may not be exactly the
+/// same.
///
-/// Since the size of the "feature" field could be either u16 or u32, and
-/// "padlen" presence is unknown, we attempt to parse .altinstructions section
-/// using all possible combinations (four at this time). Since we validate the
-/// contents of the section and its size, the detection works quite well.
-/// Still, we leave the user the opportunity to specify these features on the
-/// command line and skip the guesswork.
+/// To parse entries we only need to know the entry size and offset of
+/// the field 'instrlen'.
Error LinuxKernelRewriter::readAltInstructions() {
AltInstrSection = BC.getUniqueSectionByName(".altinstructions");
if (!AltInstrSection)
return Error::success();
- // Presence of "padlen" field.
- std::vector<bool> PadLenVariants;
- if (opts::AltInstHasPadLen.getNumOccurrences())
- PadLenVariants.push_back(opts::AltInstHasPadLen);
- else
- PadLenVariants = {false, true};
-
- // Size (in bytes) variants of "feature" field.
- std::vector<uint32_t> FeatureSizeVariants;
- if (opts::AltInstFeatureSize.getNumOccurrences())
- FeatureSizeVariants.push_back(opts::AltInstFeatureSize);
- else
- FeatureSizeVariants = {2, 4};
-
- for (bool AltInstHasPadLen : PadLenVariants) {
- for (uint32_t AltInstFeatureSize : FeatureSizeVariants) {
- LLVM_DEBUG({
- dbgs() << "BOLT-DEBUG: trying AltInstHasPadLen = " << AltInstHasPadLen
- << "; AltInstFeatureSize = " << AltInstFeatureSize << ";\n";
- });
- if (Error E = tryReadAltInstructions(AltInstFeatureSize, AltInstHasPadLen,
- /*ParseOnly*/ true)) {
- consumeError(std::move(E));
- continue;
- }
-
- LLVM_DEBUG(dbgs() << "Matched .altinstructions format\n");
-
- if (!opts::AltInstHasPadLen.getNumOccurrences())
- BC.outs() << "BOLT-INFO: setting --" << opts::AltInstHasPadLen.ArgStr
- << '=' << AltInstHasPadLen << '\n';
-
- if (!opts::AltInstFeatureSize.getNumOccurrences())
- BC.outs() << "BOLT-INFO: setting --" << opts::AltInstFeatureSize.ArgStr
- << '=' << AltInstFeatureSize << '\n';
-
- return tryReadAltInstructions(AltInstFeatureSize, AltInstHasPadLen,
- /*ParseOnly*/ false);
+ unsigned AltInstrEntrySize{0};
+ unsigned AltInstrEntryInstrlenOffset{0};
+
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (LinuxKernelVersion >= LKVersion(6, 3)) {
+ AltInstrEntrySize = 18;
+ AltInstrEntryInstrlenOffset = 16;
+ } else if (LinuxKernelVersion >= LKVersion(5, 10, 133)) {
+ AltInstrEntrySize = 12;
+ AltInstrEntryInstrlenOffset = 10;
+ } else {
+ AltInstrEntrySize = 13;
+ AltInstrEntryInstrlenOffset = 10;
}
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
}
- // We couldn't match the format. Read again to properly propagate the error
- // to the user.
- return tryReadAltInstructions(opts::AltInstFeatureSize,
- opts::AltInstHasPadLen, /*ParseOnly*/ false);
-}
+ BC.outs() << "BOLT-INFO: AltInstrEntrySize = " << AltInstrEntrySize
+ << ", AltInstrEntryInstrlenOffset = " << AltInstrEntryInstrlenOffset
+ << "\n";
-Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
- bool AltInstHasPadLen,
- bool ParseOnly) {
AddressExtractor AE(
AltInstrSection->getContents(), AltInstrSection->getAddress(),
BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
AddressExtractor::Cursor Cursor(0);
uint64_t EntryID = 0;
while (Cursor && !AE.eof(Cursor)) {
- const uint64_t OrgInstAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t AltInstAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t Feature = AE.getUnsigned(Cursor, AltInstFeatureSize);
- const uint8_t OrgSize = AE.getU8(Cursor);
- const uint8_t AltSize = AE.getU8(Cursor);
+ ++EntryID;
+ AltInstrEntries.push_back(AltInstrEntry());
+ AltInstrEntry &Entry = AltInstrEntries.back();
- // Older kernels may have the padlen field.
- const uint8_t PadLen = AltInstHasPadLen ? AE.getU8(Cursor) : 0;
+ Entry.Offset = Cursor.tell();
+ Entry.OrgInstrAddr = AE.getPCRelAddress32(Cursor);
+ Entry.AltInstrAddr = AE.getPCRelAddress32(Cursor);
+ Cursor.seek(Cursor.tell() + AltInstrEntryInstrlenOffset - 8);
+
+ Entry.Instrlen = AE.getU8(Cursor);
+ Entry.Replacementlen = AE.getU8(Cursor);
+ Cursor.seek(Cursor.tell() + AltInstrEntrySize -
+ (AltInstrEntryInstrlenOffset + 2));
if (!Cursor)
return createStringError(
@@ -1495,57 +1477,51 @@ Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
"out of bounds while reading .altinstructions: %s",
toString(Cursor.takeError()).c_str());
- ++EntryID;
-
if (opts::DumpAltInstructions) {
BC.outs() << "Alternative instruction entry: " << EntryID
- << "\n\tOrg: 0x" << Twine::utohexstr(OrgInstAddress)
- << "\n\tAlt: 0x" << Twine::utohexstr(AltInstAddress)
- << "\n\tFeature: 0x" << Twine::utohexstr(Feature)
- << "\n\tOrgSize: " << (int)OrgSize
- << "\n\tAltSize: " << (int)AltSize << '\n';
- if (AltInstHasPadLen)
- BC.outs() << "\tPadLen: " << (int)PadLen << '\n';
+ << "\n\tOrg: 0x" << Twine::utohexstr(Entry.OrgInstrAddr)
+ << "\n\tAlt: 0x" << Twine::utohexstr(Entry.AltInstrAddr)
+ << "\n\tInstrlen: " << (int)Entry.Instrlen
+ << "\n\tReplacementlen: " << (int)Entry.Replacementlen << '\n';
}
- if (AltSize > OrgSize)
+ if (Entry.Replacementlen > Entry.Instrlen)
return createStringError(errc::executable_format_error,
"error reading .altinstructions");
- BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(OrgInstAddress);
+ BinaryFunction *BF =
+ BC.getBinaryFunctionContainingAddress(Entry.OrgInstrAddr);
if (!BF && opts::Verbosity) {
BC.outs() << "BOLT-INFO: no function matches address 0x"
- << Twine::utohexstr(OrgInstAddress)
+ << Twine::utohexstr(Entry.OrgInstrAddr)
<< " of instruction from .altinstructions\n";
}
BinaryFunction *AltBF =
- BC.getBinaryFunctionContainingAddress(AltInstAddress);
- if (!ParseOnly && AltBF && BC.shouldEmit(*AltBF)) {
- BC.errs()
- << "BOLT-WARNING: alternative instruction sequence found in function "
- << *AltBF << '\n';
+ BC.getBinaryFunctionContainingAddress(Entry.AltInstrAddr);
+ if (AltBF) {
+ if (BC.isX86() &&
+ !AltBF->getOneName().starts_with(".altinstr_replacement"))
+ BC.errs() << "BOLT-WARNING: alternative instruction sequence found in "
+ "function "
+ << *AltBF << '\n';
AltBF->setIgnored();
}
if (!BF || !BF->hasInstructions())
continue;
- if (OrgInstAddress + OrgSize > BF->getAddress() + BF->getSize())
+ if (Entry.OrgInstrAddr + Entry.Instrlen > BF->getAddress() + BF->getSize())
return createStringError(errc::executable_format_error,
"error reading .altinstructions");
MCInst *Inst =
- BF->getInstructionAtOffset(OrgInstAddress - BF->getAddress());
+ BF->getInstructionAtOffset(Entry.OrgInstrAddr - BF->getAddress());
if (!Inst)
return createStringError(errc::executable_format_error,
"no instruction at address 0x%" PRIx64
" referenced by .altinstructions entry %d",
- OrgInstAddress, EntryID);
-
- if (ParseOnly)
- continue;
-
+ Entry.OrgInstrAddr, EntryID);
// There could be more than one alternative instruction sequences for the
// same original instruction. Annotate each alternative separately.
std::string AnnotationName = "AltInst";
@@ -1558,18 +1534,15 @@ Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
// Annotate all instructions from the original sequence. Note that it's not
// the most efficient way to look for instructions in the address range,
// but since alternative instructions are uncommon, it will do for now.
- for (uint32_t Offset = 1; Offset < OrgSize; ++Offset) {
- Inst = BF->getInstructionAtOffset(OrgInstAddress + Offset -
+ for (uint32_t Offset = 1; Offset < Entry.Instrlen; ++Offset) {
+ Inst = BF->getInstructionAtOffset(Entry.OrgInstrAddr + Offset -
BF->getAddress());
if (Inst)
BC.MIB->addAnnotation(*Inst, AnnotationName, EntryID);
}
}
-
- if (!ParseOnly)
- BC.outs() << "BOLT-INFO: parsed " << EntryID
- << " alternative instruction entries\n";
-
+ BC.outs() << "BOLT-INFO: parsed " << EntryID
+ << " alternative instruction entries\n";
return Error::success();
}
@@ -1691,6 +1664,8 @@ Error LinuxKernelRewriter::readPCIFixupTable() {
/// byte of the sequence with int3 before proceeding with actual code
/// replacement.
Error LinuxKernelRewriter::readStaticKeysJumpTable() {
+ LongJumpLabels = BC.isX86() && LinuxKernelVersion < LKVersion(5, 14);
+
const BinaryData *StaticKeysJumpTable =
BC.getBinaryDataByName("__start___jump_table");
if (!StaticKeysJumpTable)
@@ -1762,6 +1737,9 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
if (!BF || !BC.shouldEmit(*BF))
continue;
+ assert(BF->getOriginSection() &&
+ "the function did not originate from the file");
+ Info.BF = BF;
MCInst *Inst = BF->getInstructionAtOffset(JumpAddress - BF->getAddress());
if (!Inst)
@@ -1783,7 +1761,19 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
JumpAddress);
const uint64_t Size = BC.computeInstructionSize(*Inst);
- if (Size != 2 && Size != 5) {
+
+ auto checkSize = [this, Size]() {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (LongJumpLabels)
+ return Size == 5;
+ return Size == 2 || Size == 5;
+ default:
+ return false;
+ }
+ };
+
+ if (!checkSize()) {
return createStringError(
errc::executable_format_error,
"unexpected static keys jump size at address 0x%" PRIx64,
@@ -1805,7 +1795,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
// by the kernel patching code. Newer kernels can work with both short
// and long branches. The code for long conditional branch is larger
// than unconditional one, so we are pessimistic in our estimations.
- if (opts::LongJumpLabels)
+ if (LongJumpLabels)
BC.MIB->createLongCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get());
else
BC.MIB->createCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get());
@@ -1832,7 +1822,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
if (!BC.MIB->getOffset(*Inst))
BC.MIB->setOffset(*Inst, JumpAddress - BF->getAddress());
- if (opts::LongJumpLabels)
+ if (LongJumpLabels)
BC.MIB->setSize(*Inst, 5);
}
@@ -1865,21 +1855,27 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
const_cast<MCSymbol *>(BC.MIB->getTargetSymbol(Inst));
assert(Target && "Target symbol should be set.");
- const JumpInfoEntry &Info = JumpInfo[EntryID - 1];
+ JumpInfoEntry &Info = JumpInfo[EntryID - 1];
const bool IsBranch = Info.Likely ^ Info.InitValue;
uint32_t Size = *BC.MIB->getSize(Inst);
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
- llvm_unreachable("Wrong size for static keys jump instruction.");
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Size == 2)
+ ++NumShort;
+ else if (Size == 5)
+ ++NumLong;
+ else
+ llvm_unreachable("Wrong size for static keys jump instruction.");
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
MCInst NewInst;
// Replace the instruction with unconditional jump even if it needs to
// be nop in the binary.
- if (opts::LongJumpLabels) {
+ if (LongJumpLabels) {
BC.MIB->createLongUncondBranch(NewInst, Target, BC.Ctx.get());
} else {
// Newer kernels can handle short and long jumps for static keys.
@@ -1893,20 +1889,21 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
// Mark the instruction for nop conversion.
if (!IsBranch)
- NopIDs.insert(EntryID);
+ Info.Nop = true;
- MCSymbol *Label =
- BC.MIB->getOrCreateInstLabel(Inst, "__SK_", BC.Ctx.get());
+ Info.JumpInstLabel = BC.MIB->getOrCreateInstLabel(
+ Inst, formatv("__bolt.static_key_{0:X+8}_", EntryID), BC.Ctx.get(),
+ false);
// Create a relocation against the label.
const uint64_t EntryOffset = StaticKeysJumpTableAddress -
StaticKeysJumpSection->getAddress() +
(EntryID - 1) * 16;
- StaticKeysJumpSection->addRelocation(EntryOffset, Label,
- ELF::R_X86_64_PC32,
+ StaticKeysJumpSection->addRelocation(EntryOffset, Info.JumpInstLabel,
+ Relocation::getPC32(),
/*Addend*/ 0);
- StaticKeysJumpSection->addRelocation(EntryOffset + 4, Target,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ StaticKeysJumpSection->addRelocation(
+ EntryOffset + 4, Target, Relocation::getPC32(), /*Addend*/ 0);
}
}
}
@@ -1922,69 +1919,67 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
if (!StaticKeysJumpSection || !StaticKeysJumpSection->isFinalized())
return Error::success();
- const uint64_t SectionAddress = StaticKeysJumpSection->getAddress();
- AddressExtractor AE(StaticKeysJumpSection->getOutputContents(),
- SectionAddress, BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- AddressExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
- const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table");
- uint32_t EntryID = 0;
uint64_t NumShort = 0;
uint64_t NumLong = 0;
- while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) {
- const uint64_t JumpAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t TargetAddress = AE.getPCRelAddress32(Cursor);
- const uint64_t KeyAddress = AE.getPCRelAddress64(Cursor);
-
- // Consume the status of the cursor.
- if (!Cursor)
- return createStringError(errc::executable_format_error,
- "out of bounds while updating static keys: %s",
- toString(Cursor.takeError()).c_str());
-
- ++EntryID;
-
- LLVM_DEBUG({
- dbgs() << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress)
- << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress)
- << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n';
- });
- (void)TargetAddress;
- (void)KeyAddress;
-
- BinaryFunction *BF =
- BC.getBinaryFunctionContainingAddress(JumpAddress,
- /*CheckPastEnd*/ false,
- /*UseMaxSize*/ true);
- assert(BF && "Cannot get function for modified static key.");
+ for (JumpInfoEntry &Info : JumpInfo) {
+ MCSymbol *Label = Info.JumpInstLabel;
+ if (!Label)
+ continue;
- if (!BF->isEmitted())
+ BinaryFunction *BF = Info.BF;
+ if (!BF || !BF->isEmitted())
continue;
- // Disassemble instruction to collect stats even if nop-conversion is
- // unnecessary.
- MutableArrayRef<uint8_t> Contents = MutableArrayRef<uint8_t>(
- reinterpret_cast<uint8_t *>(BF->getImageAddress()), BF->getImageSize());
- assert(Contents.size() && "Non-empty function image expected.");
+ std::optional<uint64_t> JumpAddress = lookupSymbol(Label->getName());
+ assert(JumpAddress && "missing static key jump instruction label");
+
+ uint64_t ContentsAddress{0};
+ uint64_t ContentsSize{0};
+ MutableArrayRef<uint8_t> Contents;
+
+ if (!BC.HasRelocations) {
+ const FunctionFragment *FF =
+ BF->getFunctionFragmentForOutputAddress(*JumpAddress);
+ assert(FF && "Can not get fragment for jump address");
+
+ ContentsAddress = FF->getAddress();
+ ContentsSize = FF->getImageSize();
+ Contents = MutableArrayRef<uint8_t>(FF->getOutputData(), ContentsSize);
+ } else {
+ ErrorOr<BinarySection &> Sec =
+ BC.getSectionForOutputAddress(*JumpAddress);
+ assert(Sec && "Can not get section for jump address.");
+
+ ContentsAddress = Sec->getOutputAddress();
+ ContentsSize = Sec->getOutputSize();
+ Contents = MutableArrayRef<uint8_t>(Sec->getOutputData(), ContentsSize);
+ }
MCInst Inst;
uint64_t Size;
- const uint64_t JumpOffset = JumpAddress - BF->getAddress();
+ const uint64_t JumpOffset = *JumpAddress - ContentsAddress;
if (!BC.DisAsm->getInstruction(Inst, Size, Contents.slice(JumpOffset), 0,
nulls())) {
llvm_unreachable("Unable to disassemble jump instruction.");
}
assert(BC.MIB->isBranch(Inst) && "Branch instruction expected.");
-
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
- llvm_unreachable("Unexpected size for static keys jump instruction.");
+ assert(JumpOffset + Size <= ContentsAddress + ContentsSize);
+
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Size == 2)
+ ++NumShort;
+ else if (Size == 5)
+ ++NumLong;
+ else
+ llvm_unreachable("Unexpected size for static keys jump instruction.");
+ break;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
// Check if we need to convert jump instruction into a nop.
- if (!NopIDs.contains(EntryID))
+ if (!Info.Nop)
continue;
SmallString<15> NopCode;
@@ -2003,6 +1998,6 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createLinuxKernelRewriter(BinaryContext &BC) {
- return std::make_unique<LinuxKernelRewriter>(BC);
+llvm::bolt::createLinuxKernelRewriter(RewriteInstance &RI) {
+ return std::make_unique<LinuxKernelRewriter>(RI);
}
diff --git a/bolt/lib/Rewrite/MetadataRewriter.cpp b/bolt/lib/Rewrite/MetadataRewriter.cpp
new file mode 100644
index 0000000000000..962e7704167b9
--- /dev/null
+++ b/bolt/lib/Rewrite/MetadataRewriter.cpp
@@ -0,0 +1,20 @@
+//===------------ bolt/Rewrite/MetadataRewriter.cpp -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Rewrite/MetadataRewriter.h"
+#include "bolt/Rewrite/RewriteInstance.h"
+
+using namespace llvm;
+using namespace bolt;
+
+MetadataRewriter::MetadataRewriter(StringRef Name, RewriteInstance &RI)
+ : Name(Name), RI(RI), BC(*RI.BC) {}
+
+std::optional<uint64_t> MetadataRewriter::lookupSymbol(const StringRef Name) {
+ return RI.Linker->lookupSymbol(Name);
+}
diff --git a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
index 9d6e914624a33..3e4d839b4cbc4 100644
--- a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
+++ b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
@@ -80,8 +80,8 @@ class PseudoProbeRewriter final : public MetadataRewriter {
std::shared_ptr<MCPseudoProbeDecoder> ProbeDecoderPtr;
public:
- PseudoProbeRewriter(BinaryContext &BC)
- : MetadataRewriter("pseudo-probe-rewriter", BC),
+ PseudoProbeRewriter(RewriteInstance &RI)
+ : MetadataRewriter("pseudo-probe-rewriter", RI),
ProbeDecoderPtr(std::make_shared<MCPseudoProbeDecoder>()) {
BC.setPseudoProbeDecoder(ProbeDecoderPtr);
}
@@ -447,6 +447,6 @@ void PseudoProbeRewriter::encodePseudoProbes() {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createPseudoProbeRewriter(BinaryContext &BC) {
- return std::make_unique<PseudoProbeRewriter>(BC);
+llvm::bolt::createPseudoProbeRewriter(RewriteInstance &RI) {
+ return std::make_unique<PseudoProbeRewriter>(RI);
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 70a9f084f009b..4b161af262209 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -541,9 +541,13 @@ Error RewriteInstance::discoverStorage() {
BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
Phdr.p_vaddr, Phdr.p_memsz, Phdr.p_offset,
Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
- if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
- Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
- BC->IsLinuxKernel = true;
+ switch (BC->TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
+ BC->IsLinuxKernel = true;
+ break;
+ default:;
+ }
break;
case ELF::PT_INTERP:
BC->HasInterpHeader = true;
@@ -3208,13 +3212,13 @@ void RewriteInstance::preprocessProfileData() {
void RewriteInstance::initializeMetadataManager() {
if (BC->IsLinuxKernel)
- MetadataManager.registerRewriter(createLinuxKernelRewriter(*BC));
+ MetadataManager.registerRewriter(createLinuxKernelRewriter(*this));
- MetadataManager.registerRewriter(createBuildIDRewriter(*BC));
+ MetadataManager.registerRewriter(createBuildIDRewriter(*this));
- MetadataManager.registerRewriter(createPseudoProbeRewriter(*BC));
+ MetadataManager.registerRewriter(createPseudoProbeRewriter(*this));
- MetadataManager.registerRewriter(createSDTRewriter(*BC));
+ MetadataManager.registerRewriter(createSDTRewriter(*this));
}
void RewriteInstance::processSectionMetadata() {
@@ -3878,6 +3882,7 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
<< " to 0x" << Twine::utohexstr(Function.getAddress())
<< '\n');
MapSection(*FuncSection, Function.getAddress());
+ Function.getLayout().getMainFragment().setAddress(Function.getAddress());
Function.setImageAddress(FuncSection->getAllocAddress());
Function.setImageSize(FuncSection->getOutputSize());
assert(Function.getImageSize() <= Function.getMaxSize() &&
diff --git a/bolt/lib/Rewrite/SDTRewriter.cpp b/bolt/lib/Rewrite/SDTRewriter.cpp
index a3928c554ad66..2558403fac763 100644
--- a/bolt/lib/Rewrite/SDTRewriter.cpp
+++ b/bolt/lib/Rewrite/SDTRewriter.cpp
@@ -55,7 +55,8 @@ class SDTRewriter final : public MetadataRewriter {
void printSDTMarkers() const;
public:
- SDTRewriter(StringRef Name, BinaryContext &BC) : MetadataRewriter(Name, BC) {}
+ SDTRewriter(StringRef Name, RewriteInstance &RI)
+ : MetadataRewriter(Name, RI) {}
Error preCFGInitializer() override;
@@ -173,6 +174,6 @@ void SDTRewriter::printSDTMarkers() const {
} // namespace
std::unique_ptr<MetadataRewriter>
-llvm::bolt::createSDTRewriter(BinaryContext &BC) {
- return std::make_unique<SDTRewriter>("sdt-rewriter", BC);
+llvm::bolt::createSDTRewriter(RewriteInstance &RI) {
+ return std::make_unique<SDTRewriter>("sdt-rewriter", RI);
}
diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s
index 83d2cd0634d08..3e299685cf5bb 100644
--- a/bolt/test/X86/linux-alt-instruction.s
+++ b/bolt/test/X86/linux-alt-instruction.s
@@ -6,31 +6,9 @@
# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-cfg --alt-inst-feature-size=2 -o %t.out \
+# RUN: llvm-bolt %t.exe --print-cfg -o %t.out \
# RUN: | FileCheck %s
-## Older kernels used to have padlen field in alt_instr. Check compatibility.
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown --defsym PADLEN=1 \
-# RUN: %s -o %t.padlen.o
-# RUN: %clang %cflags -nostdlib %t.padlen.o -o %t.padlen.exe \
-# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.padlen.exe --print-cfg --alt-inst-has-padlen -o %t.padlen.out \
-# RUN: | FileCheck %s
-
-## Check with a larger size of "feature" field in alt_instr.
-
-# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown \
-# RUN: --defsym FEATURE_SIZE_4=1 %s -o %t.fs4.o
-# RUN: %clang %cflags -nostdlib %t.fs4.o -o %t.fs4.exe \
-# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.fs4.exe --print-cfg --alt-inst-feature-size=4 -o %t.fs4.out \
-# RUN: | FileCheck %s
-
-## Check that out-of-bounds read is handled properly.
-
-# RUN: not llvm-bolt %t.fs4.exe --alt-inst-feature-size=2 -o %t.fs4.out
-
## Check that BOLT automatically detects structure fields in .altinstructions.
# RUN: llvm-bolt %t.exe --print-cfg -o %t.out | FileCheck %s
@@ -78,11 +56,7 @@ _start:
.long .L0 - . # org instruction
.long .A0 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x72 # feature flags
-.else
.word 0x72 # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .A1 - .A0 # alt size
.ifdef PADLEN
@@ -91,11 +65,7 @@ _start:
.long .L0 - . # org instruction
.long .A1 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x3b # feature flags
-.else
.word 0x3b # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .A2 - .A1 # alt size
.ifdef PADLEN
@@ -104,11 +74,7 @@ _start:
.long .L0 - . # org instruction
.long .A2 - . # alt instruction
-.ifdef FEATURE_SIZE_4
- .long 0x110 # feature flags
-.else
.word 0x110 # feature flags
-.endif
.byte .L1 - .L0 # org size
.byte .Ae - .A2 # alt size
.ifdef PADLEN
@@ -148,7 +114,7 @@ _start:
.globl linux_banner
.type linux_banner, @object
linux_banner:
- .string "Linux version 6.6.61\n"
+ .string "Linux version 6.1\n"
.size linux_banner, . - linux_banner
## Fake Linux Kernel sections.
More information about the llvm-commits
mailing list