[llvm] [BOLT][RISCV] Improve relocations, jump tables, and split-function handling (PR #213919)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 19:55:14 PDT 2026
https://github.com/Thrrreeee updated https://github.com/llvm/llvm-project/pull/213919
>From d0797ec87fcf78623f5c622dd3ee98225c9420b7 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Wed, 15 Jul 2026 20:19:35 +0800
Subject: [PATCH 01/10] [BOLT][RISCV] Handle static IFUNC entries in .iplt
Recognize R_RISCV_IRELATIVE and RISC-V .iplt entries, preserve IFUNC PLT aliases, and register resolver addends as secondary function entry points after function boundaries are finalized.
Preserve resolver-relative offsets when adding secondary entries so non-zero IRELATIVE addends remain valid.
---
bolt/include/bolt/Rewrite/RewriteInstance.h | 3 +-
bolt/lib/Core/Relocation.cpp | 6 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 76 +++++++++++++++++++--
bolt/test/RISCV/ifunc.s | 40 +++++++++++
4 files changed, 119 insertions(+), 6 deletions(-)
create mode 100644 bolt/test/RISCV/ifunc.s
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index a624c056ada14..53e173fce929f 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -564,7 +564,8 @@ class RewriteInstance {
{".plt"}, {".plt.got"}, {".iplt"}, {nullptr}};
/// RISCV PLT sections.
- const PLTSectionInfo RISCV_PLTSections[2] = {{".plt"}, {nullptr}};
+ const PLTSectionInfo RISCV_PLTSections[3] = {{".plt"}, {".iplt", 16},
+ {nullptr}};
/// Return PLT information for a section with \p SectionName or nullptr
/// if the section is not PLT.
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index b0f6b6ce0eddc..a2ea552acbedc 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -130,6 +130,7 @@ static bool isSupportedRISCV(uint32_t Type) {
case ELF::R_RISCV_TPREL_ADD:
case ELF::R_RISCV_TPREL_LO12_I:
case ELF::R_RISCV_TPREL_LO12_S:
+ case ELF::R_RISCV_IRELATIVE:
case ELFReserved::R_RISCV_TPREL_I:
case ELFReserved::R_RISCV_TPREL_S:
return true;
@@ -244,6 +245,9 @@ static size_t getSizeForTypeRISCV(uint32_t Type) {
case ELF::R_RISCV_TLS_GD_HI20:
// See extractValueRISCV for why this is necessary.
return 8;
+ case ELF::R_RISCV_IRELATIVE:
+ // R_RISCV_IRELATIVE operates on a wordclass field.
+ return Relocation::Arch == Triple::riscv64 ? 8 : 4;
}
}
@@ -871,7 +875,7 @@ bool Relocation::isIRelative(uint32_t Type) {
return Type == ELF::R_AARCH64_IRELATIVE;
case Triple::riscv64:
case Triple::riscv32:
- llvm_unreachable("not implemented");
+ return Type == ELF::R_RISCV_IRELATIVE;
case Triple::x86_64:
return Type == ELF::R_X86_64_IRELATIVE;
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 51b19db0cbd9b..67a19dcc47124 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1385,6 +1385,30 @@ void RewriteInstance::discoverFileObjects() {
adjustFunctionBoundaries(MarkerSymbols);
splitUnmarkedTailFunctions(MarkerSymbols);
+ // R_RISCV_IRELATIVE addends name resolver entry points. LLD may
+ // canonicalize the only IFUNC symbol to the IPLT entry, leaving the resolver
+ // without a symbol. Function sizes are not final when dynamic relocations
+ // are first read, so record these secondary entries after boundary
+ // adjustment.
+ if (BC->isRISCV()) {
+ for (const BinarySection &Section : BC->allocatableSections()) {
+ for (const Relocation &Rel : Section.dynamicRelocations()) {
+ if (!Rel.isIRelative() || !Rel.Addend)
+ continue;
+ BinaryFunction *BF = BC->getBinaryFunctionContainingAddress(Rel.Addend);
+ if (!BF || BF->getAddress() == Rel.Addend)
+ continue;
+ if (BF->isInConstantIsland(Rel.Addend)) {
+ BC->errs() << "BOLT-ERROR: IFUNC resolver at 0x"
+ << Twine::utohexstr(Rel.Addend)
+ << " is in constant island of function " << *BF << '\n';
+ exit(1);
+ }
+ BF->addEntryPointAtOffset(Rel.Addend - BF->getAddress());
+ }
+ }
+ }
+
// Annotate functions with code/data markers in AArch64
for (auto &[Address, Type] : MarkerSymbols) {
auto *BF = BC->getBinaryFunctionContainingAddress(Address,
@@ -1880,7 +1904,7 @@ void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,
MCSymbol *Symbol = Rel->Symbol;
if (!Symbol) {
- if (BC->isRISCV() || !Rel->Addend || !Rel->isIRelative())
+ if (!Rel->Addend || !Rel->isIRelative())
return;
// IFUNC trampoline without symbol
@@ -1904,6 +1928,23 @@ void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,
else
BF->addAlternativeName(Symbol->getName().str() + "@PLT");
setPLTSymbol(BF, Symbol->getName());
+
+ if (Rel->isIRelative()) {
+ auto ResolverSyms = FileSymRefs.equal_range(Rel->Addend);
+ for (const SymbolRef &AliasSymbol : llvm::make_second_range(
+ llvm::make_range(ResolverSyms.first, ResolverSyms.second))) {
+ if (ELFSymbolRef(AliasSymbol).getELFType() != ELF::STT_GNU_IFUNC)
+ continue;
+ StringRef AliasName = cantFail(AliasSymbol.getName());
+ const std::string PLTName = AliasName.str() + "@PLT";
+ if (!BC->getBinaryDataByName(PLTName)) {
+ BF->addAlternativeName(PLTName);
+ BC->registerNameAtAddress(PLTName, EntryAddress, 0, EntrySize,
+ Section->getAlignment());
+ }
+ setPLTSymbol(BF, AliasName);
+ }
+ }
}
void RewriteInstance::disassemblePLTInstruction(const BinarySection &Section,
@@ -1992,8 +2033,9 @@ void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) {
}
};
- // Skip the first special entry since no relocation points to it.
- uint64_t InstrOffset = 32;
+ // Regular .plt has a first special entry with no relocations pointing to it,
+ // while static IFUNC .iplt entries start at the beginning of the section.
+ uint64_t InstrOffset = Section.getName() == ".iplt" ? 0 : 32;
while (InstrOffset < SectionSize) {
InstructionListType Instructions;
@@ -2778,7 +2820,10 @@ bool RewriteInstance::analyzeRelocation(
// Section symbols are marked as ST_Debug.
IsSectionRelocation = (cantFail(Symbol.getType()) == SymbolRef::ST_Debug);
// Check for PLT entry registered with symbol name
- if (!SymbolAddress && !IsWeakReference(Symbol) &&
+ const bool IsRISCVIFuncPLT =
+ BC->isRISCV() && RType == ELF::R_RISCV_CALL_PLT &&
+ ELFSymbolRef(Symbol).getELFType() == ELF::STT_GNU_IFUNC;
+ if ((!SymbolAddress || IsRISCVIFuncPLT) && !IsWeakReference(Symbol) &&
(IsAArch64 || BC->isRISCV())) {
const BinaryData *BD = BC->getPLTBinaryDataByName(SymbolName);
SymbolAddress = BD ? BD->getAddress() : 0;
@@ -6426,6 +6471,29 @@ uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {
}
uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) {
+ // Resolve secondary function entry points before the exact-address lookup.
+ // getBinaryFunctionAtAddress() can map a BinaryData symbol at a secondary
+ // entry back to its parent function and would then return the parent's main
+ // output address, losing the entry-point offset.
+ if (const BinaryFunction *BF =
+ BC->getBinaryFunctionContainingAddress(OldAddress)) {
+ if (BF->isEmitted() && BF->isMultiEntry()) {
+ uint64_t EntryAddress = 0;
+ BF->forEachEntryPoint([&](uint64_t Offset, const MCSymbol *Symbol) {
+ if (Offset && BF->getAddress() + Offset == OldAddress) {
+ if (auto SymbolInfo = Linker->lookupSymbolInfo(Symbol->getName()))
+ EntryAddress = SymbolInfo->Address;
+ else
+ EntryAddress = BF->translateInputToOutputAddress(OldAddress);
+ return false;
+ }
+ return true;
+ });
+ if (EntryAddress)
+ return EntryAddress;
+ }
+ }
+
if (uint64_t Function = getNewFunctionAddress(OldAddress))
return Function;
diff --git a/bolt/test/RISCV/ifunc.s b/bolt/test/RISCV/ifunc.s
new file mode 100644
index 0000000000000..587c117ed0943
--- /dev/null
+++ b/bolt/test/RISCV/ifunc.s
@@ -0,0 +1,40 @@
+## Check that BOLT recognizes a non-preemptible IFUNC IPLT entry and tracks
+## the resolver when the linker canonicalizes the IFUNC symbol to the entry.
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax -o %t.o %s
+# RUN: ld.lld -pie -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm --print-only=_start 2>&1 \
+# RUN: | FileCheck --check-prefix=BOLT %s
+# RUN: llvm-readelf -r -s %t.bolt | FileCheck --check-prefix=ELF %s
+## RV32 static binaries use a 32-bit wordclass IRELATIVE field. Also verify
+## that a resolver at a secondary entry point retains its +4 offset.
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax -o %t.32.o %s
+# RUN: ld.lld -q -o %t.32.exe %t.32.o
+# RUN: llvm-bolt %t.32.exe -o %t.32.bolt
+# RUN: llvm-readelf -r -s %t.32.bolt | FileCheck --check-prefix=RV32 %s
+
+# BOLT: Binary Function "_start
+# BOLT: auipc a0, %pcrel_hi(__BOLT_PSEUDO_.iplt)
+# BOLT-NOT: unable to get new address corresponding to input address
+# ELF: R_RISCV_IRELATIVE{{.*}}400044
+# ELF: FUNC{{.*}}ifunc0
+# RV32: R_RISCV_IRELATIVE{{.*}}400044
+# RV32: FUNC{{.*}}ifunc0
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+1:
+ auipc a0, %pcrel_hi(ifunc0)
+ addi a0, a0, %pcrel_lo(1b)
+
+ .globl func
+ .type func, @function
+func:
+ ret
+
+ .globl ifunc0
+ .type ifunc0, @gnu_indirect_function
+ifunc0:
+ ret
>From fb48ee8415e7acfb1e47e6aa76bc2f9fdd90fea9 Mon Sep 17 00:00:00 2001
From: shijinrui <shijinrui at bytedance.com>
Date: Wed, 19 Aug 2026 15:40:11 +0800
Subject: [PATCH 02/10] [BOLT][RISCV] Add target symbolizer for relocations
Move RISC-V relocation sanitization out of BinaryFunction::disassemble() and into RISCVMCSymbolizer. Teach the RISC-V disassembler to invoke the symbolizer for UImm20 and SImm12 operands so HI20/LO12 and GOT pairs remain valid when code moves.
Keep pcrel_lo labels attached to their referenced instructions, handle relocations left inside instructions after R_RISCV_ALIGN processing, and cover both RV32 and RV64 with encoding checks.
Co-authored-by: Thrrreeeee <1379998393 at qq.com>
---
bolt/lib/Core/BinaryFunction.cpp | 51 +-----
bolt/lib/Core/Relocation.cpp | 2 +-
bolt/lib/Target/RISCV/CMakeLists.txt | 2 +
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 7 +
bolt/lib/Target/RISCV/RISCVMCSymbolizer.cpp | 159 ++++++++++++++++++
bolt/lib/Target/RISCV/RISCVMCSymbolizer.h | 55 ++++++
bolt/test/RISCV/reloc-bb-split-rv32.s | 12 +-
bolt/test/RISCV/reloc-got-moved-rv32.s | 35 ++++
bolt/test/RISCV/reloc-got-moved.s | 35 ++++
bolt/test/RISCV/reloc-got.s | 26 ++-
bolt/test/RISCV/reloc-pcrel-moved.s | 31 ++++
bolt/test/RISCV/reloc-pcrel-rv32.s | 11 +-
bolt/test/RISCV/reloc-pcrel.s | 3 +-
.../RISCV/Disassembler/RISCVDisassembler.cpp | 28 +++
llvm/lib/Target/RISCV/RISCVInstrInfo.td | 2 +
15 files changed, 382 insertions(+), 77 deletions(-)
create mode 100644 bolt/lib/Target/RISCV/RISCVMCSymbolizer.cpp
create mode 100644 bolt/lib/Target/RISCV/RISCVMCSymbolizer.h
create mode 100644 bolt/test/RISCV/reloc-got-moved-rv32.s
create mode 100644 bolt/test/RISCV/reloc-got-moved.s
create mode 100644 bolt/test/RISCV/reloc-pcrel-moved.s
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index a81fa2f45c206..6924d3b9684e8 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -33,6 +33,7 @@
#include "llvm/MC/MCInstPrinter.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/Object/ELF.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@@ -1330,13 +1331,6 @@ Error BinaryFunction::disassemble() {
// basic block.
Labels[0] = Ctx->createNamedTempSymbol("BB0");
- // Map offsets in the function to a label that should always point to the
- // corresponding instruction. This is used for labels that shouldn't point to
- // the start of a basic block but always to a specific instruction. This is
- // used, for example, on RISC-V where %pcrel_lo relocations point to the
- // corresponding %pcrel_hi.
- LabelsMapType InstructionLabels;
-
uint64_t Size = 0; // instruction size
for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) {
MCInst Instruction;
@@ -1501,42 +1495,6 @@ Error BinaryFunction::disassemble() {
if (BC.isAArch64())
handleAArch64IndirectCall(Instruction, Offset);
}
- } else if (BC.isRISCV()) {
- // Check if there's a relocation associated with this instruction.
- for (auto Itr = Relocations.lower_bound(Offset),
- ItrE = Relocations.lower_bound(Offset + Size);
- Itr != ItrE; ++Itr) {
- const Relocation &Relocation = Itr->second;
- MCSymbol *Symbol = Relocation.Symbol;
-
- if (Relocation::isInstructionReference(Relocation.Type)) {
- uint64_t RefOffset = Relocation.Value - getAddress();
- LabelsMapType::iterator LI = InstructionLabels.find(RefOffset);
-
- if (LI == InstructionLabels.end()) {
- Symbol = BC.Ctx->createNamedTempSymbol();
- InstructionLabels.emplace(RefOffset, Symbol);
- } else {
- Symbol = LI->second;
- }
- }
-
- uint64_t Addend = Relocation.Addend;
-
- // For GOT relocations, create a reference against GOT entry ignoring
- // the relocation symbol.
- if (Relocation::isGOT(Relocation.Type)) {
- assert(Relocation::isPCRelative(Relocation.Type) &&
- "GOT relocation must be PC-relative on RISC-V");
- Symbol = BC.registerNameAtAddress("__BOLT_got_zero", 0, 0, 0);
- Addend = Relocation.Value + Relocation.Offset + getAddress();
- }
- int64_t Value = Relocation.Value;
- const bool Result = BC.MIB->replaceImmWithSymbolRef(
- Instruction, Symbol, Addend, Ctx.get(), Value, Relocation.Type);
- (void)Result;
- assert(Result && "cannot replace immediate with relocation");
- }
}
add_instruction:
@@ -1578,13 +1536,6 @@ Error BinaryFunction::disassemble() {
// Scope-boundary markers are only consulted while assigning offsets above.
DebugScopeBoundaryOffsets.clear();
- for (auto [Offset, Label] : InstructionLabels) {
- InstrMapType::iterator II = Instructions.find(Offset);
- assert(II != Instructions.end() && "reference to non-existing instruction");
-
- BC.MIB->setInstLabel(II->second, Label);
- }
-
// Reset symbolizer for the disassembler.
BC.SymbolicDisAsm->setSymbolizer(nullptr);
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index a2ea552acbedc..983f458e45997 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -896,7 +896,7 @@ bool Relocation::isTLS(uint32_t Type) {
}
bool Relocation::isInstructionReference(uint32_t Type) {
- if (Arch != Triple::riscv64)
+ if (Arch != Triple::riscv64 && Arch != Triple::riscv32)
return false;
switch (Type) {
diff --git a/bolt/lib/Target/RISCV/CMakeLists.txt b/bolt/lib/Target/RISCV/CMakeLists.txt
index 45645a98d132f..e7fa950de29df 100644
--- a/bolt/lib/Target/RISCV/CMakeLists.txt
+++ b/bolt/lib/Target/RISCV/CMakeLists.txt
@@ -1,5 +1,6 @@
set(LLVM_LINK_COMPONENTS
MC
+ MCDisassembler
Support
RISCVDesc
)
@@ -19,6 +20,7 @@ endif()
add_llvm_library(LLVMBOLTTargetRISCV
RISCVMCPlusBuilder.cpp
+ RISCVMCSymbolizer.cpp
NO_EXPORT
DISABLE_LLVM_LINK_LLVM_DYLIB
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 1511e4744124a..c6c04ca88aa7a 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -12,6 +12,7 @@
#include "MCTargetDesc/RISCVMCAsmInfo.h"
#include "MCTargetDesc/RISCVMCTargetDesc.h"
+#include "RISCVMCSymbolizer.h"
#include "bolt/Core/MCPlusBuilder.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/MC/MCContext.h"
@@ -39,6 +40,12 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
public:
using MCPlusBuilder::MCPlusBuilder;
+ std::unique_ptr<MCSymbolizer>
+ createTargetSymbolizer(BinaryFunction &Function,
+ bool CreateNewSymbols) const override {
+ return std::make_unique<RISCVMCSymbolizer>(Function, CreateNewSymbols);
+ }
+
bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
CompFuncTy Comp) const override {
const auto &RISCVExprA = cast<MCSpecifierExpr>(A);
diff --git a/bolt/lib/Target/RISCV/RISCVMCSymbolizer.cpp b/bolt/lib/Target/RISCV/RISCVMCSymbolizer.cpp
new file mode 100644
index 0000000000000..9def396f788b2
--- /dev/null
+++ b/bolt/lib/Target/RISCV/RISCVMCSymbolizer.cpp
@@ -0,0 +1,159 @@
+//===- bolt/Target/RISCV/RISCVMCSymbolizer.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCVMCSymbolizer.h"
+#include "bolt/Core/BinaryContext.h"
+#include "bolt/Core/BinaryFunction.h"
+#include "bolt/Core/MCPlusBuilder.h"
+#include "bolt/Core/Relocation.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCInst.h"
+
+#define DEBUG_TYPE "bolt-symbolizer"
+
+namespace llvm {
+namespace bolt {
+
+RISCVMCSymbolizer::RISCVMCSymbolizer(BinaryFunction &Function,
+ bool CreateNewSymbols)
+ : MCSymbolizer(*Function.getBinaryContext().Ctx, nullptr),
+ Function(Function), CreateNewSymbols(CreateNewSymbols) {
+ if (!CreateNewSymbols)
+ return;
+
+ // Discover instruction references before decoding starts. This lets us
+ // attach a label while decoding the referenced %pcrel_hi instruction even
+ // though its %pcrel_lo user is normally decoded later.
+ for (uint64_t SearchOffset = 0; SearchOffset < Function.getSize();) {
+ const Relocation *Rel =
+ Function.getRelocationInRange(SearchOffset, Function.getSize());
+ if (!Rel)
+ break;
+
+ if (Relocation::isInstructionReference(Rel->Type)) {
+ assert(Rel->Value >= Function.getAddress() &&
+ Rel->Value < Function.getAddress() + Function.getSize() &&
+ "RISC-V instruction reference outside of function");
+ InstructionLabels.try_emplace(Rel->Value - Function.getAddress(),
+ nullptr);
+ }
+
+ SearchOffset = Rel->Offset + 1;
+ }
+}
+
+RISCVMCSymbolizer::~RISCVMCSymbolizer() {}
+
+MCSymbol *RISCVMCSymbolizer::getOrCreateInstructionLabel(uint64_t Offset) {
+ auto [It, Inserted] = InstructionLabels.try_emplace(Offset, nullptr);
+ (void)Inserted;
+ if (!It->second)
+ It->second = Ctx.createNamedTempSymbol();
+ return It->second;
+}
+
+uint64_t RISCVMCSymbolizer::getGOTValue(const Relocation &Rel) const {
+ BinaryContext &BC = Function.getBinaryContext();
+ const uint64_t HiAddress = Function.getAddress() + Rel.Offset;
+
+ // A GOT high relocation records a combined high/low value. Locate the low
+ // relocation by its reference back to this AUIPC instead of assuming that
+ // the low instruction is adjacent.
+ for (uint64_t SearchOffset = 0; SearchOffset < Function.getSize();) {
+ const Relocation *LoRel =
+ Function.getRelocationInRange(SearchOffset, Function.getSize());
+ if (!LoRel)
+ break;
+ SearchOffset = LoRel->Offset + 1;
+
+ if (!Relocation::isInstructionReference(LoRel->Type) ||
+ LoRel->Value != HiAddress)
+ continue;
+
+ ErrorOr<uint64_t> HiContents = BC.getUnsignedValueAtAddress(HiAddress, 4);
+ ErrorOr<uint64_t> LoContents =
+ BC.getUnsignedValueAtAddress(Function.getAddress() + LoRel->Offset,
+ Relocation::getSizeForType(LoRel->Type));
+ assert(HiContents && LoContents &&
+ "cannot read RISC-V GOT relocation pair");
+
+ return Relocation::extractValue(ELF::R_RISCV_PCREL_HI20, *HiContents,
+ HiAddress) +
+ Relocation::extractValue(LoRel->Type, *LoContents,
+ Function.getAddress() + LoRel->Offset);
+ }
+
+ return Rel.Value;
+}
+
+bool RISCVMCSymbolizer::tryAddingSymbolicOperand(
+ MCInst &Inst, raw_ostream &CStream, int64_t Value, uint64_t InstAddress,
+ bool IsBranch, uint64_t ImmOffset, uint64_t ImmSize, uint64_t InstSize) {
+ BinaryContext &BC = Function.getBinaryContext();
+ MCContext *Ctx = BC.Ctx.get();
+ const uint64_t InstOffset = InstAddress - Function.getAddress();
+
+ // Branches and calls are resolved by BinaryFunction's target-independent
+ // control-flow handling.
+ if (BC.MIB->isBranch(Inst) || BC.MIB->isCall(Inst))
+ return false;
+
+ // Linker processing of R_RISCV_ALIGN can leave emitted relocations at an
+ // offset inside the instruction they apply to. Match the whole instruction
+ // range, as BinaryFunction::disassemble() did before this target-specific
+ // handling moved into the symbolizer.
+ const Relocation *Rel =
+ Function.getRelocationInRange(InstOffset, InstOffset + InstSize);
+ if (!Rel)
+ return false;
+
+ MCSymbol *Symbol = Rel->Symbol;
+ uint64_t Addend = Rel->Addend;
+
+ if (Relocation::isInstructionReference(Rel->Type)) {
+ if (!CreateNewSymbols)
+ return false;
+ Symbol = getOrCreateInstructionLabel(Rel->Value - Function.getAddress());
+ // The input addend reflects the original AUIPC location. The label now
+ // follows the instruction, so the assembler must derive the low bits from
+ // its new location.
+ Addend = 0;
+ }
+
+ // GOT high relocations name the object stored in the GOT, not the GOT entry
+ // addressed by AUIPC. Preserve the actual entry address using a zero-based
+ // symbol, as the RISC-V emitter reuses the input GOT.
+ if (Relocation::isGOT(Rel->Type)) {
+ assert(Relocation::isPCRelative(Rel->Type) &&
+ "GOT relocation must be PC-relative on RISC-V");
+ Symbol = BC.registerNameAtAddress("__BOLT_got_zero", 0, 0, 0);
+ Addend = getGOTValue(*Rel) + InstAddress;
+ }
+
+ assert(Symbol && "RISC-V relocation without a symbol");
+ const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, *Ctx);
+ if (Addend)
+ Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(Addend, *Ctx),
+ *Ctx);
+ Inst.addOperand(MCOperand::createExpr(
+ BC.MIB->getTargetExprFor(Inst, Expr, *Ctx, Rel->Type)));
+
+ // MC annotations must follow every real operand. Attach the instruction
+ // label only after the symbolized immediate has been appended.
+ if (InstructionLabels.find(InstOffset) != InstructionLabels.end())
+ BC.MIB->setInstLabel(Inst, getOrCreateInstructionLabel(InstOffset));
+
+ return true;
+}
+
+void RISCVMCSymbolizer::tryAddingPcLoadReferenceComment(raw_ostream &CStream,
+ int64_t Value,
+ uint64_t Address) {}
+
+} // namespace bolt
+} // namespace llvm
diff --git a/bolt/lib/Target/RISCV/RISCVMCSymbolizer.h b/bolt/lib/Target/RISCV/RISCVMCSymbolizer.h
new file mode 100644
index 0000000000000..d3b628daae746
--- /dev/null
+++ b/bolt/lib/Target/RISCV/RISCVMCSymbolizer.h
@@ -0,0 +1,55 @@
+//===- bolt/Target/RISCV/RISCVMCSymbolizer.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_TARGET_RISCV_RISCVMCSYMBOLIZER_H
+#define BOLT_TARGET_RISCV_RISCVMCSYMBOLIZER_H
+
+#include "bolt/Core/BinaryFunction.h"
+#include "llvm/MC/MCDisassembler/MCSymbolizer.h"
+#include <map>
+
+namespace llvm {
+namespace bolt {
+
+class RISCVMCSymbolizer : public MCSymbolizer {
+protected:
+ BinaryFunction &Function;
+ bool CreateNewSymbols{true};
+
+ /// Map function offsets referenced by %pcrel_lo relocations to labels that
+ /// must remain attached to the corresponding %pcrel_hi instructions.
+ std::map<uint64_t, MCSymbol *> InstructionLabels;
+
+ MCSymbol *getOrCreateInstructionLabel(uint64_t Offset);
+
+ /// Return the complete PC-relative value for a GOT relocation. The value
+ /// recorded when relocations are read assumes that the low instruction
+ /// immediately follows AUIPC. Reconstruct it from the matching low
+ /// relocation so linker scheduling does not affect symbolization.
+ uint64_t getGOTValue(const Relocation &Rel) const;
+
+public:
+ RISCVMCSymbolizer(BinaryFunction &Function, bool CreateNewSymbols = true);
+
+ RISCVMCSymbolizer(const RISCVMCSymbolizer &) = delete;
+ RISCVMCSymbolizer &operator=(const RISCVMCSymbolizer &) = delete;
+ ~RISCVMCSymbolizer() override;
+
+ bool tryAddingSymbolicOperand(MCInst &Inst, raw_ostream &CStream,
+ int64_t Value, uint64_t Address, bool IsBranch,
+ uint64_t Offset, uint64_t OpSize,
+ uint64_t InstSize) override;
+
+ void tryAddingPcLoadReferenceComment(raw_ostream &CStream, int64_t Value,
+ uint64_t Address) override;
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif
diff --git a/bolt/test/RISCV/reloc-bb-split-rv32.s b/bolt/test/RISCV/reloc-bb-split-rv32.s
index 0ad3168fb983d..a434f5c71bd63 100644
--- a/bolt/test/RISCV/reloc-bb-split-rv32.s
+++ b/bolt/test/RISCV/reloc-bb-split-rv32.s
@@ -20,10 +20,10 @@ _start:
/// basic block should start there.
// CHECK-LABEL: {{^}}.LBB00
// CHECK: nop
-// CHECK-LABEL: {{^}}.Ltmp0
-// CHECK: auipc t0, %pcrel_hi(d)
-// CHECK-NEXT: lw t0, %pcrel_lo({{.*}})(t0)
-// CHECK-NEXT: j .Ltmp0
+// CHECK: {{^}}[[BRANCH_LABEL:.Ltmp[0-9]+]]
+// CHECK: auipc t0, %pcrel_hi(d) # Label: [[HI_LABEL:.Ltmp[0-9]+]]
+// CHECK-NEXT: lw t0, %pcrel_lo([[HI_LABEL]])(t0)
+// CHECK-NEXT: j [[BRANCH_LABEL]]
nop
1:
auipc t0, %pcrel_hi(d)
@@ -34,8 +34,8 @@ _start:
/// start there.
// CHECK-LABEL: {{^}}.LFT0
// CHECK: nop
-// CHECK: auipc t0, %pcrel_hi(d)
-// CHECK-NEXT: lw t0, %pcrel_lo({{.*}})(t0)
+// CHECK: auipc t0, %pcrel_hi(d) # Label: [[SECOND_HI:.Ltmp[0-9]+]]
+// CHECK-NEXT: lw t0, %pcrel_lo([[SECOND_HI]])(t0)
// CHECK-NEXT: ret
nop
1:
diff --git a/bolt/test/RISCV/reloc-got-moved-rv32.s b/bolt/test/RISCV/reloc-got-moved-rv32.s
new file mode 100644
index 0000000000000..f4076a35112e1
--- /dev/null
+++ b/bolt/test/RISCV/reloc-got-moved-rv32.s
@@ -0,0 +1,35 @@
+## Check that the RV32 R_RISCV_GOT_HI20/%pcrel_lo pair is rebuilt when the
+## matching low instruction is not immediately after AUIPC.
+
+# RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -reorder-functions=cdsort --check-encoding
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s
+
+# CHECK: Disassembly of section .text:
+# CHECK: <_start>:
+# CHECK-NEXT: auipc a0, 0xffc12
+# CHECK-NEXT: li a1, 0x7
+# CHECK-NEXT: li a2, 0x9
+# CHECK-NEXT: lw a0, 0x128(a0)
+# CHECK-NEXT: ret
+
+ .data
+ .p2align 12
+ .globl d
+d:
+ .word 0
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ nop
+1:
+ auipc a0, %got_pcrel_hi(d)
+ addi a1, zero, 7
+ addi a2, zero, 9
+ lw a0, %pcrel_lo(1b)(a0)
+ ret
+ .reloc 0, R_RISCV_NONE
+ .size _start, .-_start
diff --git a/bolt/test/RISCV/reloc-got-moved.s b/bolt/test/RISCV/reloc-got-moved.s
new file mode 100644
index 0000000000000..c11155c657210
--- /dev/null
+++ b/bolt/test/RISCV/reloc-got-moved.s
@@ -0,0 +1,35 @@
+## Check that R_RISCV_GOT_HI20 relocations are re-encoded correctly when the
+## matching %pcrel_lo is not in the instruction immediately after the AUIPC.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -reorder-functions=cdsort --check-encoding
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s
+
+# CHECK: Disassembly of section .text:
+# CHECK: <_start>:
+# CHECK-NEXT: auipc a0, 0xffc12
+# CHECK-NEXT: li a1, 0x7
+# CHECK-NEXT: li a2, 0x9
+# CHECK-NEXT: ld a0, 0x1e0(a0)
+# CHECK-NEXT: ret
+
+ .data
+ .p2align 12
+ .globl d
+d:
+ .dword 0
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ nop
+1:
+ auipc a0, %got_pcrel_hi(d)
+ addi a1, zero, 7
+ addi a2, zero, 9
+ ld a0, %pcrel_lo(1b)(a0)
+ ret
+ .reloc 0, R_RISCV_NONE
+ .size _start, .-_start
diff --git a/bolt/test/RISCV/reloc-got.s b/bolt/test/RISCV/reloc-got.s
index 1860da3e05a3b..905d8451a6365 100644
--- a/bolt/test/RISCV/reloc-got.s
+++ b/bolt/test/RISCV/reloc-got.s
@@ -1,5 +1,6 @@
// RUN: %clang %cflags64 -o %t %s
-// RUN: llvm-bolt --print-cfg --print-only=_start -o %t.null %t \
+// RUN: llvm-bolt --check-encoding --print-cfg --print-only=_start \
+// RUN: -o %t.null %t \
// RUN: | FileCheck %s
.data
@@ -22,30 +23,27 @@ _start:
auipc t0, %got_pcrel_hi(d)
ld t0, %pcrel_lo(1b)(t0)
-/// An unrelated instruction sits between the AUIPC and its load.
-// FIXME: The AUIPC below should also use __BOLT_got_zero+[[GOT]], but BOLT
-// takes the low part from the ADDI instead of from the load that names the
-// AUIPC's label.
-// CHECK-NOT: __BOLT_got_zero+[[GOT]])
-// CHECK: addi t2, t2, 0x7ff
-// CHECK-NEXT: ld t1, %pcrel_lo({{\.Ltmp[0-9]+}})(t1)
+/// An unrelated instruction can sit between the AUIPC and its load. The
+/// symbolizer locates the low relocation through its reference to the AUIPC.
+// CHECK: auipc t1, %pcrel_hi(__BOLT_got_zero+[[GOT]]) # Label: [[HI2:\.Ltmp[0-9]+]]
+// CHECK-NEXT: addi t2, t2, 0x7ff
+// CHECK-NEXT: ld t1, %pcrel_lo([[HI2]])(t1)
2:
auipc t1, %got_pcrel_hi(d)
addi t2, t2, 2047
ld t1, %pcrel_lo(2b)(t1)
j .L1
.L2:
+// CHECK: ld t1, %pcrel_lo([[HI3:\.Ltmp[0-9]+]])(t1)
+// CHECK-NEXT: j
ld t1, %pcrel_lo(3f)(t1)
j .Lexit
.L1:
nop
-/// The load lives in another basic block, so nothing follows the AUIPC but
-/// the terminator.
-// FIXME: The AUIPC below should also use __BOLT_got_zero+[[GOT]], but BOLT
-// takes the low part from the jump.
+/// The low relocation can also precede the AUIPC in output basic-block order.
// CHECK: nop
-// CHECK-NOT: __BOLT_got_zero+[[GOT]])
-// CHECK: j
+// CHECK-NEXT: auipc t1, %pcrel_hi(__BOLT_got_zero+[[GOT]]) # Label: [[HI3]]
+// CHECK-NEXT: j
3:
auipc t1, %got_pcrel_hi(d)
j .L2
diff --git a/bolt/test/RISCV/reloc-pcrel-moved.s b/bolt/test/RISCV/reloc-pcrel-moved.s
new file mode 100644
index 0000000000000..0b69ac2ca48be
--- /dev/null
+++ b/bolt/test/RISCV/reloc-pcrel-moved.s
@@ -0,0 +1,31 @@
+## Check that R_RISCV_PCREL_LO12 relocations are re-encoded relative to the
+## moved AUIPC instruction instead of retaining the input addend.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -reorder-functions=cdsort --check-encoding
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s
+
+# CHECK: Disassembly of section .text:
+# CHECK: <_start>:
+# CHECK-NEXT: auipc a0, 0xffc13
+# CHECK-NEXT: ld a0, 0x0(a0)
+# CHECK-NEXT: ret
+
+ .data
+ .p2align 12
+ .globl d
+d:
+ .dword 0
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ nop
+1:
+ auipc a0, %pcrel_hi(d)
+ ld a0, %pcrel_lo(1b)(a0)
+ ret
+ .reloc 0, R_RISCV_NONE
+ .size _start, .-_start
diff --git a/bolt/test/RISCV/reloc-pcrel-rv32.s b/bolt/test/RISCV/reloc-pcrel-rv32.s
index 2e386b022b8f8..f78a9d75bf796 100644
--- a/bolt/test/RISCV/reloc-pcrel-rv32.s
+++ b/bolt/test/RISCV/reloc-pcrel-rv32.s
@@ -2,7 +2,8 @@
// RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj -o %t.o %s
// RUN: ld.lld -q -o %t %t.o
-// RUN: llvm-bolt --print-cfg --print-only=_start -o %t.null %t \
+// RUN: llvm-bolt --check-encoding --print-cfg --print-only=_start \
+// RUN: -o %t.null %t \
// RUN: | FileCheck %s
.data
@@ -17,11 +18,11 @@ d:
// CHECK: Binary Function "_start" after building cfg {
_start:
nop // Here to not make the _start and .Ltmp0 symbols coincide
-// CHECK: auipc t0, %pcrel_hi(d)
-// CHECK-NEXT: lw t0, %pcrel_lo({{.*}})(t0)
+// CHECK: auipc t0, %pcrel_hi(d) # Label: [[HI_LABEL:.Ltmp[0-9]+]]
+// CHECK-NEXT: lw t0, %pcrel_lo([[HI_LABEL]])(t0)
lw t0, d
-// CHECK: auipc t1, %pcrel_hi(d)
-// CHECK-NEXT: sw t0, %pcrel_lo({{.*}})(t1)
+// CHECK: auipc t1, %pcrel_hi(d) # Label: [[SECOND_HI:.Ltmp[0-9]+]]
+// CHECK-NEXT: sw t0, %pcrel_lo([[SECOND_HI]])(t1)
sw t0, d, t1
ret
.size _start, .-_start
diff --git a/bolt/test/RISCV/reloc-pcrel.s b/bolt/test/RISCV/reloc-pcrel.s
index 5320c6d12e5cd..12324b4e112c9 100644
--- a/bolt/test/RISCV/reloc-pcrel.s
+++ b/bolt/test/RISCV/reloc-pcrel.s
@@ -1,5 +1,6 @@
// RUN: %clang %cflags64 -o %t %s
-// RUN: llvm-bolt --print-cfg --print-only=_start -o %t.null %t \
+// RUN: llvm-bolt --check-encoding --print-cfg --print-only=_start \
+// RUN: -o %t.null %t \
// RUN: | FileCheck %s
.data
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 3b3eb5195a9b4..92379c2630b84 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -403,6 +403,34 @@ static DecodeStatus decodeSImmOperand(MCInst &Inst, uint32_t Imm,
return MCDisassembler::Success;
}
+static DecodeStatus decodeSImm12Operand(MCInst &Inst, uint32_t Imm,
+ int64_t Address,
+ const MCDisassembler *Decoder) {
+ assert(isUInt<12>(Imm) && "Invalid immediate");
+ const int64_t Value = SignExtend64<12>(Imm);
+ // A register-relative immediate is not itself a branch target. Use a
+ // one-byte operand size to prevent MCExternalSymbolizer from guessing that
+ // an unsymbolized immediate is an absolute address; target symbolizers use
+ // the instruction relocation and do not depend on this operand size.
+ if (!Decoder->tryAddingSymbolicOperand(Inst, Value, Address,
+ /*IsBranch=*/false,
+ /*Offset=*/0, /*OpSize=*/1,
+ /*InstSize=*/4))
+ Inst.addOperand(MCOperand::createImm(Value));
+ return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeUImm20Operand(MCInst &Inst, uint32_t Imm,
+ int64_t Address,
+ const MCDisassembler *Decoder) {
+ assert(isUInt<20>(Imm) && "Invalid immediate");
+ if (!Decoder->tryAddingSymbolicOperand(Inst, Imm, Address, /*IsBranch=*/false,
+ /*Offset=*/0,
+ /*OpSize=*/1, /*InstSize=*/4))
+ Inst.addOperand(MCOperand::createImm(Imm));
+ return MCDisassembler::Success;
+}
+
template <unsigned N>
static DecodeStatus decodeSImmNonZeroOperand(MCInst &Inst, uint32_t Imm,
int64_t Address,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 5be74d729c5b0..85a339e28d588 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -400,6 +400,7 @@ def uimm64 : RISCVUImmOp<64>;
def simm12 : RISCVSImmLeafOp<12>;
class Simm12LoOp : RISCVSImmLeafOp<12> {
+ let DecoderMethod = "decodeSImm12Operand";
let MCOperandPredicate = [{
int64_t Imm;
if (MCOp.evaluateAsConstantImm(Imm))
@@ -455,6 +456,7 @@ def bare_simm13_lsb0 : BareSImm13Lsb0MaybeSym,
def bare_simm13_lsb0_bb : BareSImm13Lsb0MaybeSym;
class UImm20OperandMaybeSym : RISCVUImmOp<20> {
+ let DecoderMethod = "decodeUImm20Operand";
let MCOperandPredicate = [{
int64_t Imm;
if (MCOp.evaluateAsConstantImm(Imm))
>From d6af0db4542e00cba8ff29c30cc11b2c5b533417 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Wed, 15 Jul 2026 20:37:33 +0800
Subject: [PATCH 03/10] [BOLT][RISCV] Preserve branches when rescanning ignored
code
Create relocations for RISC-V MC fixups and re-encode JAL, branch, and compressed control-flow immediates while preserving the original instruction bits. Avoid registering invalid secondary entries in constant islands, and use relaxable PseudoTAIL instructions when rebuilding long tail calls.
---
bolt/include/bolt/Core/Relocation.h | 6 +-
bolt/lib/Core/BinaryContext.cpp | 13 ++--
bolt/lib/Core/BinaryFunction.cpp | 16 +++++
bolt/lib/Core/BinarySection.cpp | 25 +++++--
bolt/lib/Core/Relocation.cpp | 68 +++++++++++++++++-
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 72 ++++++++++++++++++-
.../test/RISCV/constant-island-entry-rescan.s | 44 ++++++++++++
bolt/test/RISCV/ignored-func-short-branch.s | 58 +++++++++++++++
8 files changed, 284 insertions(+), 18 deletions(-)
create mode 100644 bolt/test/RISCV/constant-island-entry-rescan.s
create mode 100644 bolt/test/RISCV/ignored-func-short-branch.s
diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h
index 9bc94d6484b70..8ee3e2587cb9c 100644
--- a/bolt/include/bolt/Core/Relocation.h
+++ b/bolt/include/bolt/Core/Relocation.h
@@ -94,8 +94,10 @@ class Relocation {
/// Skip relocations that we don't want to handle in BOLT
static bool skipRelocationType(uint32_t Type);
- /// Adjust value depending on relocation type (make it PC relative or not).
- static uint64_t encodeValue(uint32_t Type, uint64_t Value, uint64_t PC);
+ /// Encode \p Value according to the relocation type. \p OldValue is used only
+ /// by RISC-V instruction relocations that preserve non-immediate bits.
+ static uint64_t encodeValue(uint32_t Type, uint64_t Value, uint64_t PC,
+ uint64_t OldValue = 0);
/// Return true if there are enough bits to encode the relocation value.
static bool canEncodeValue(uint32_t Type, uint64_t Value, uint64_t PC);
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index d911f8191f791..2877a795d7bd2 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -573,12 +573,13 @@ MCSymbol *BinaryContext::handleExternalBranchTarget(uint64_t Address,
<< Twine::utohexstr(Address) << "; ignoring both functions\n";
IsValid = false;
}
- if (Target.isInConstantIsland(Address)) {
- this->errs() << "BOLT-WARNING: ignoring entry point at address 0x"
- << Twine::utohexstr(Address)
- << " in constant island of function " << Target << '\n';
- IsValid = false;
- }
+ }
+
+ if (Target.isInConstantIsland(Address)) {
+ this->errs() << "BOLT-WARNING: ignoring entry point at address 0x"
+ << Twine::utohexstr(Address)
+ << " in constant island of function " << Target << '\n';
+ IsValid = false;
}
if (!IsValid) {
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 6924d3b9684e8..6c5e833539f90 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1835,6 +1835,22 @@ bool BinaryFunction::scanExternalRefs() {
continue;
}
+ if (BC.isRISCV()) {
+ switch (Rel->Type) {
+ default:
+ break;
+ case ELF::R_RISCV_BRANCH:
+ case ELF::R_RISCV_JAL:
+ case ELF::R_RISCV_RVC_BRANCH:
+ case ELF::R_RISCV_RVC_JUMP:
+ if (BinaryFunction *TargetBF = BC.getFunctionForSymbol(Rel->Symbol)) {
+ TargetBF->setNeedsPatch(true);
+ continue;
+ }
+ break;
+ }
+ }
+
if (BC.isAArch64()) {
// Allow the relocation to be skipped in case of the overflow during the
// relocation value encoding.
diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp
index a8620ba83ebfb..9205523661357 100644
--- a/bolt/lib/Core/BinarySection.cpp
+++ b/bolt/lib/Core/BinarySection.cpp
@@ -191,18 +191,31 @@ void BinarySection::flushPendingRelocations(raw_fd_ostream &OS,
++SkippedPendingRelocations;
continue;
}
+
+ const size_t RelocSize = Relocation::getSizeForType(Reloc.Type);
+ uint64_t OldValue = 0;
+ if (Reloc.Offset + RelocSize <= getContents().size()) {
+ ArrayRef<uint8_t> Bytes(reinterpret_cast<const uint8_t *>(
+ getContents().data() + Reloc.Offset),
+ RelocSize);
+ if (BC.AsmInfo->isLittleEndian()) {
+ for (unsigned I = 0; I < Bytes.size(); ++I)
+ OldValue |= uint64_t(Bytes[I]) << (I * 8);
+ } else {
+ for (uint8_t Byte : Bytes)
+ OldValue = (OldValue << 8) | Byte;
+ }
+ }
Value = Relocation::encodeValue(Reloc.Type, Value,
- SectionAddress + Reloc.Offset);
+ SectionAddress + Reloc.Offset, OldValue);
safePWrite(OS, reinterpret_cast<const char *>(&Value),
- Relocation::getSizeForType(Reloc.Type),
- SectionFileOffset + Reloc.Offset);
+ RelocSize, SectionFileOffset + Reloc.Offset);
LLVM_DEBUG(
dbgs() << "BOLT-DEBUG: writing value 0x" << Twine::utohexstr(Value)
- << " of size " << Relocation::getSizeForType(Reloc.Type)
- << " at section offset 0x" << Twine::utohexstr(Reloc.Offset)
- << " address 0x"
+ << " of size " << RelocSize << " at section offset 0x"
+ << Twine::utohexstr(Reloc.Offset) << " address 0x"
<< Twine::utohexstr(SectionAddress + Reloc.Offset)
<< " file offset 0x"
<< Twine::utohexstr(SectionFileOffset + Reloc.Offset) << '\n';);
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index 983f458e45997..b8354ac91278a 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -343,13 +343,74 @@ static uint64_t canEncodeValueRISCV(uint32_t Type, uint64_t Value,
}
}
-static uint64_t encodeValueRISCV(uint32_t Type, uint64_t Value, uint64_t PC) {
+static uint64_t encodeValueRISCV(uint32_t Type, uint64_t Value, uint64_t PC,
+ uint64_t OldValue) {
+ auto encodePCRel = [&](uint32_t Type, uint64_t Value) -> uint64_t {
+ const int64_t PCRelValue =
+ static_cast<int64_t>(Value) - static_cast<int64_t>(PC);
+ assert((PCRelValue & 0x1) == 0 && "RISC-V branch target is misaligned");
+ const uint64_t EncValue = static_cast<uint64_t>(PCRelValue);
+ switch (Type) {
+ default:
+ llvm_unreachable("unsupported relocation");
+ case ELF::R_RISCV_BRANCH: {
+ assert(isInt<13>(PCRelValue) && "RISC-V branch target out of range");
+ const uint64_t Sbit = (EncValue >> 12) & 0x1;
+ const uint64_t Hi1 = (EncValue >> 11) & 0x1;
+ const uint64_t Mid6 = (EncValue >> 5) & 0x3f;
+ const uint64_t Lo4 = (EncValue >> 1) & 0xf;
+ return (OldValue & 0x01fff07f) | (Sbit << 31) | (Mid6 << 25) |
+ (Lo4 << 8) | (Hi1 << 7);
+ }
+ case ELF::R_RISCV_JAL: {
+ assert(isInt<21>(PCRelValue) && "RISC-V jump target out of range");
+ const uint64_t Sbit = (EncValue >> 20) & 0x1;
+ const uint64_t Hi8 = (EncValue >> 12) & 0xff;
+ const uint64_t Mid1 = (EncValue >> 11) & 0x1;
+ const uint64_t Lo10 = (EncValue >> 1) & 0x3ff;
+ return (OldValue & 0xfff) | (Sbit << 31) | (Lo10 << 21) | (Mid1 << 20) |
+ (Hi8 << 12);
+ }
+ case ELF::R_RISCV_RVC_BRANCH: {
+ assert(isInt<9>(PCRelValue) &&
+ "RISC-V compressed branch target out of range");
+ const uint64_t Bit8 = (EncValue >> 8) & 0x1;
+ const uint64_t Bit7_6 = (EncValue >> 6) & 0x3;
+ const uint64_t Bit5 = (EncValue >> 5) & 0x1;
+ const uint64_t Bit4_3 = (EncValue >> 3) & 0x3;
+ const uint64_t Bit2_1 = (EncValue >> 1) & 0x3;
+ return (OldValue & 0xe383) | (Bit8 << 12) | (Bit4_3 << 10) |
+ (Bit7_6 << 5) | (Bit2_1 << 3) | (Bit5 << 2);
+ }
+ case ELF::R_RISCV_RVC_JUMP: {
+ assert(isInt<12>(PCRelValue) &&
+ "RISC-V compressed jump target out of range");
+ const uint64_t Bit11 = (EncValue >> 11) & 0x1;
+ const uint64_t Bit4 = (EncValue >> 4) & 0x1;
+ const uint64_t Bit9_8 = (EncValue >> 8) & 0x3;
+ const uint64_t Bit10 = (EncValue >> 10) & 0x1;
+ const uint64_t Bit6 = (EncValue >> 6) & 0x1;
+ const uint64_t Bit7 = (EncValue >> 7) & 0x1;
+ const uint64_t Bit3_1 = (EncValue >> 1) & 0x7;
+ const uint64_t Bit5 = (EncValue >> 5) & 0x1;
+ return (OldValue & 0xe003) | (Bit11 << 12) | (Bit4 << 11) |
+ (Bit9_8 << 9) | (Bit10 << 8) | (Bit6 << 7) | (Bit7 << 6) |
+ (Bit3_1 << 3) | (Bit5 << 2);
+ }
+ }
+ };
+
switch (Type) {
default:
llvm_unreachable("unsupported relocation");
case ELF::R_RISCV_32:
case ELF::R_RISCV_64:
break;
+ case ELF::R_RISCV_BRANCH:
+ case ELF::R_RISCV_JAL:
+ case ELF::R_RISCV_RVC_BRANCH:
+ case ELF::R_RISCV_RVC_JUMP:
+ return encodePCRel(Type, Value);
}
return Value;
}
@@ -782,7 +843,8 @@ bool Relocation::skipRelocationType(uint32_t Type) {
}
}
-uint64_t Relocation::encodeValue(uint32_t Type, uint64_t Value, uint64_t PC) {
+uint64_t Relocation::encodeValue(uint32_t Type, uint64_t Value, uint64_t PC,
+ uint64_t OldValue) {
switch (Arch) {
default:
llvm_unreachable("Unsupported architecture");
@@ -790,7 +852,7 @@ uint64_t Relocation::encodeValue(uint32_t Type, uint64_t Value, uint64_t PC) {
return encodeValueAArch64(Type, Value, PC);
case Triple::riscv64:
case Triple::riscv32:
- return encodeValueRISCV(Type, Value, PC);
+ return encodeValueRISCV(Type, Value, PC, OldValue);
case Triple::x86_64:
return encodeValueX86(Type, Value, PC);
}
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index c6c04ca88aa7a..eed94ff5e307d 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "MCTargetDesc/RISCVMCAsmInfo.h"
+#include "MCTargetDesc/RISCVFixupKinds.h"
#include "MCTargetDesc/RISCVMCTargetDesc.h"
#include "RISCVMCSymbolizer.h"
#include "bolt/Core/MCPlusBuilder.h"
@@ -280,7 +281,8 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
MCContext *Ctx) override {
- createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);
+ Seq.emplace_back();
+ createTailCall(Seq.back(), Target, Ctx);
}
void createTailCall(MCInst &Inst, const MCSymbol *Target,
@@ -659,6 +661,74 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
return Insts;
}
+ std::optional<Relocation>
+ createRelocation(const MCFixup &Fixup,
+ const MCAsmBackend &MAB) const override {
+ (void)MAB;
+ const uint64_t RelOffset = Fixup.getOffset();
+
+ uint32_t RelType;
+ if (mc::isRelocation(Fixup.getKind())) {
+ RelType = Fixup.getKind();
+ } else if (Fixup.isPCRel()) {
+ switch (Fixup.getKind()) {
+ default:
+ return std::nullopt;
+ case FK_Data_4:
+ RelType = ELF::R_RISCV_32_PCREL;
+ break;
+ case RISCV::fixup_riscv_pcrel_hi20:
+ RelType = ELF::R_RISCV_PCREL_HI20;
+ break;
+ case RISCV::fixup_riscv_pcrel_lo12_i:
+ RelType = ELF::R_RISCV_PCREL_LO12_I;
+ break;
+ case RISCV::fixup_riscv_pcrel_lo12_s:
+ RelType = ELF::R_RISCV_PCREL_LO12_S;
+ break;
+ case RISCV::fixup_riscv_jal:
+ RelType = ELF::R_RISCV_JAL;
+ break;
+ case RISCV::fixup_riscv_branch:
+ RelType = ELF::R_RISCV_BRANCH;
+ break;
+ case RISCV::fixup_riscv_rvc_jump:
+ RelType = ELF::R_RISCV_RVC_JUMP;
+ break;
+ case RISCV::fixup_riscv_rvc_branch:
+ RelType = ELF::R_RISCV_RVC_BRANCH;
+ break;
+ case RISCV::fixup_riscv_call:
+ case RISCV::fixup_riscv_call_plt:
+ RelType = ELF::R_RISCV_CALL_PLT;
+ break;
+ }
+ } else {
+ switch (Fixup.getKind()) {
+ default:
+ return std::nullopt;
+ case FK_Data_4:
+ RelType = ELF::R_RISCV_32;
+ break;
+ case FK_Data_8:
+ RelType = ELF::R_RISCV_64;
+ break;
+ case RISCV::fixup_riscv_hi20:
+ RelType = ELF::R_RISCV_HI20;
+ break;
+ case RISCV::fixup_riscv_lo12_i:
+ RelType = ELF::R_RISCV_LO12_I;
+ break;
+ case RISCV::fixup_riscv_lo12_s:
+ RelType = ELF::R_RISCV_LO12_S;
+ break;
+ }
+ }
+
+ auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
+ return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
+ }
+
InstructionListType createInstrIncMemory(const MCSymbol *Target,
MCContext *Ctx, bool IsLeaf,
unsigned CodePointerSize) override {
diff --git a/bolt/test/RISCV/constant-island-entry-rescan.s b/bolt/test/RISCV/constant-island-entry-rescan.s
new file mode 100644
index 0000000000000..7c49ec511e6d8
--- /dev/null
+++ b/bolt/test/RISCV/constant-island-entry-rescan.s
@@ -0,0 +1,44 @@
+# This test verifies that BOLT does not crash while rescanning references from
+# an ignored function when its branch target lies in another function's constant
+# island.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -reorder-blocks=ext-tsp \
+# RUN: -reorder-functions=cdsort -simplify-rodata-loads -plt=hot \
+# RUN: -split-eh -use-gnu-stack 2>&1 | FileCheck %s
+
+# CHECK: BOLT-WARNING: corrupted control flow detected in function source:
+# CHECK-SAME: an external branch/call targets an invalid instruction
+# CHECK-SAME: in function target at address 0x{{[0-9a-f]+}}; ignoring both functions
+# CHECK: BOLT-WARNING: ignoring entry point at address 0x{{[0-9a-f]+}} in constant island of function target
+# CHECK-NOT: cannot add entry point that points to constant data
+
+ .text
+ .globl target
+ .type target, @function
+target:
+ j after_data
+
+data_label:
+ .word 0
+
+after_data:
+ ret
+ .size target, .-target
+
+ .globl source
+ .type source, @function
+source:
+ j data_label
+ ret
+ .size source, .-source
+
+ .globl _start
+ .type _start, @function
+_start:
+ call source
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
diff --git a/bolt/test/RISCV/ignored-func-short-branch.s b/bolt/test/RISCV/ignored-func-short-branch.s
new file mode 100644
index 0000000000000..f965406732ab2
--- /dev/null
+++ b/bolt/test/RISCV/ignored-func-short-branch.s
@@ -0,0 +1,58 @@
+# This test verifies that rescanning references in an ignored RISC-V function
+# does not try to redirect short branch/jump relocations to moved code.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld -q -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -reorder-blocks=ext-tsp \
+# RUN: -reorder-functions=cdsort -simplify-rodata-loads -plt=hot \
+# RUN: -split-eh -use-gnu-stack 2>&1 | FileCheck %s
+
+# CHECK: BOLT-WARNING: corrupted control flow detected in function source:
+# CHECK: BOLT-WARNING: ignoring entry point at address 0x{{[0-9a-f]+}} in constant island of function target
+# CHECK-NOT: unsupported relocation
+# CHECK-NOT: could not find corresponding %pcrel_hi
+# CHECK-NOT: target out of range
+
+ .text
+ .globl target
+ .type target, @function
+target:
+ j after_data
+
+data_label:
+ .word 0
+
+after_data:
+ ret
+ .size target, .-target
+
+ .globl callee
+ .type callee, @function
+callee:
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ nop
+ ret
+ .size callee, .-callee
+
+ .globl source
+ .type source, @function
+source:
+ j data_label
+ beqz a0, callee
+ ret
+ .size source, .-source
+
+ .globl _start
+ .type _start, @function
+_start:
+ call source
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
>From b743916bff234dd833456d388261434886bd033d Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Wed, 15 Jul 2026 20:19:35 +0800
Subject: [PATCH 04/10] [BOLT][RISCV] Test conditional tail-call lowering
Exercise a conditional branch to another function through the post-lowering dump. Verify that the conditional edge is inverted around a generated tail call instead of being treated as an unconditional jump.
---
bolt/test/RISCV/conditional-tail-call.s | 67 ++++++-------------------
1 file changed, 15 insertions(+), 52 deletions(-)
diff --git a/bolt/test/RISCV/conditional-tail-call.s b/bolt/test/RISCV/conditional-tail-call.s
index 65817570406cc..273a3a40f452f 100644
--- a/bolt/test/RISCV/conditional-tail-call.s
+++ b/bolt/test/RISCV/conditional-tail-call.s
@@ -1,67 +1,30 @@
-// Check that all base and compressed RISC-V conditional branches targeting
-// another function are expanded to tail-call blocks, survive block reordering,
-// and are emitted with the correct target.
+## Check that a conditional branch to another function is handled as a
+## conditional tail call. The target-specific jump-to-tail-call conversion must
+## return false for conditional branches so the generic code records the CTC
+## annotation instead of marking the branch as an unconditional tail call.
-// RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
-// RUN: ld.lld -o %t %t.o
-// RUN: llvm-bolt %t -o %t.bolt --reorder-blocks=reverse --print-cfg \
-// RUN: --print-only=conditional_tail_calls 2>&1 | FileCheck %s
-// RUN: llvm-objdump -d --disassemble-symbols=conditional_tail_calls %t.bolt \
-// RUN: | FileCheck %s --check-prefix=DISASM
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld --emit-relocs -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt --print-after-lowering --print-only=_start \
+# RUN: 2>&1 | FileCheck %s
-// CHECK: Binary Function "conditional_tail_calls" after building cfg {
-// CHECK: beq a0, a1, .LTC0
-// CHECK: bne a0, a1, .LTC1
-// CHECK: blt a0, a1, .LTC2
-// CHECK: bge a0, a1, .LTC3
-// CHECK: bltu a0, a1, .LTC4
-// CHECK: bgeu a0, a1, .LTC5
-// CHECK: beqz a0, .LTC6
-// CHECK: bnez a0, .LTC7
-// CHECK: BOLT-INFO: basic block reordering modified layout of 1 functions
-
-// DISASM-LABEL: <conditional_tail_calls>:
-// DISASM-NEXT: {{.*}} beq a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} bne a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} blt a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} bge a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} bltu a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} bgeu a0, a1, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} beqz a0, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} bnez a0, {{.*}} <callee>
-// DISASM-NEXT: {{.*}} ret
+# CHECK: Binary Function "_start"
+# CHECK: bnez a0, .Ltmp[[#]]
+# CHECK: tail callee
+# CHECK: End of Function "_start"
.text
- .option rvc
-
- .globl conditional_tail_calls
- .type conditional_tail_calls, @function
- .p2align 1
-conditional_tail_calls:
- .option push
- .option exact
- beq a0, a1, callee
- bne a0, a1, callee
- blt a0, a1, callee
- bge a0, a1, callee
- bltu a0, a1, callee
- bgeu a0, a1, callee
- c.beqz a0, callee
- c.bnez a0, callee
- .option pop
- ret
- .size conditional_tail_calls, .-conditional_tail_calls
-
.globl callee
.type callee, @function
- .p2align 1
callee:
ret
.size callee, .-callee
.globl _start
.type _start, @function
- .p2align 1
_start:
+ beq a0, zero, callee
ret
.size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
>From c39bd58ef30f9526e916cc3c95959e9e30fdd458 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Wed, 15 Jul 2026 20:24:50 +0800
Subject: [PATCH 05/10] [BOLT][RISCV] Support split function fragments
Run long-branch relaxation for split RISC-V functions and route cross-fragment edges through AUIPC/JALR trampolines. Select an ABI-safe dead scratch GPR using liveness analysis and keep a function unsplit when no register is available.
Account for explicit call operands, restrict candidates to the RVE register set when applicable, and preserve mapping-symbol names across split fragments.
---
bolt/include/bolt/Core/MCPlusBuilder.h | 3 +-
bolt/lib/Passes/LongJmp.cpp | 261 ++++++++++++++----
bolt/lib/Rewrite/BinaryPassManager.cpp | 4 +
bolt/lib/Rewrite/RewriteInstance.cpp | 21 +-
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 4 +-
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 83 ++++++
.../split-functions-indirect-call-scratch.s | 43 +++
bolt/test/RISCV/split-functions-long-jump.s | 48 ++++
.../RISCV/split-functions-no-scratch-rve.s | 33 +++
bolt/test/RISCV/split-functions-no-scratch.s | 49 ++++
10 files changed, 496 insertions(+), 53 deletions(-)
create mode 100644 bolt/test/RISCV/split-functions-indirect-call-scratch.s
create mode 100644 bolt/test/RISCV/split-functions-long-jump.s
create mode 100644 bolt/test/RISCV/split-functions-no-scratch-rve.s
create mode 100644 bolt/test/RISCV/split-functions-no-scratch.s
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index be0d58af14fc4..9882980396b50 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1828,7 +1828,8 @@ class MCPlusBuilder {
}
virtual void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
- MCContext *Ctx, bool IsTailCall = false) {
+ MCContext *Ctx, bool IsTailCall = false,
+ MCPhysReg ScratchReg = 0) {
llvm_unreachable("not implemented");
}
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 30ff9dc4ddc71..93f3e2754dce5 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -685,14 +685,147 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
if (BF.isSimple() && !BF.isSplit() && BF.estimateSize() < ShortestJumpSpan)
return true;
+ DenseMap<const MCInst *, MCPhysReg> ScratchRegs;
+ if (BC.isRISCV()) {
+ const unsigned NumRegs = BC.MRI->getNumRegs();
+ DenseMap<const BinaryBasicBlock *, BitVector> LiveIns;
+ DenseMap<const BinaryBasicBlock *, BitVector> LiveOuts;
+ for (const BinaryBasicBlock &BB : BF) {
+ LiveIns.try_emplace(&BB, NumRegs, false);
+ LiveOuts.try_emplace(&BB, NumRegs, false);
+ }
+
+ BitVector ABIExitState(NumRegs, false);
+ MIB->getDefaultLiveOut(ABIExitState);
+ MIB->getCalleeSavedRegs(ABIExitState);
+
+ auto transfer = [&](const MCInst &Inst, BitVector State) {
+ if (MIB->isCFI(Inst))
+ return State;
+
+ BitVector Written(NumRegs, false);
+ BitVector Used(NumRegs, false);
+ MIB->getWrittenRegs(Inst, Written);
+ MIB->getUsedRegs(Inst, Used);
+ if (MIB->isCall(Inst)) {
+ BitVector CallClobbered(NumRegs, false);
+ MIB->getGPRegs(CallClobbered, /*IncludeAlias=*/true);
+ BitVector Preserved(NumRegs, false);
+ MIB->getCalleeSavedRegs(Preserved);
+ Preserved.flip();
+ CallClobbered &= Preserved;
+ Written |= CallClobbered;
+ Used |= MIB->getRegsUsedAsParams();
+ }
+ Written.flip();
+ State &= Written;
+ State |= Used;
+ return State;
+ };
+
+ bool Changed;
+ do {
+ Changed = false;
+ for (BinaryBasicBlock &BB : reverse(BF)) {
+ BitVector LiveOut(NumRegs, false);
+ if (BB.succ_size() == 0)
+ LiveOut = ABIExitState;
+ else
+ for (const BinaryBasicBlock *Succ : BB.successors())
+ LiveOut |= LiveIns[Succ];
+
+ BitVector LiveIn = LiveOut;
+ for (const MCInst &Inst : reverse(BB))
+ LiveIn = transfer(Inst, std::move(LiveIn));
+
+ if (LiveOuts[&BB] != LiveOut) {
+ LiveOuts[&BB] = std::move(LiveOut);
+ Changed = true;
+ }
+ if (LiveIns[&BB] != LiveIn) {
+ LiveIns[&BB] = std::move(LiveIn);
+ Changed = true;
+ }
+ }
+ } while (Changed);
+
+ bool CanRelax = true;
+ for (BinaryBasicBlock &BB : BF) {
+ BitVector Live = LiveOuts[&BB];
+ for (MCInst &Inst : reverse(BB)) {
+ if (!MIB->isBranch(Inst) || MIB->isIndirectBranch(Inst))
+ Live = transfer(Inst, std::move(Live));
+ else {
+ const MCSymbol *TargetSymbol = MIB->getTargetSymbol(Inst);
+ BinaryBasicBlock *TargetBB = BB.getSuccessor(TargetSymbol);
+ if (TargetBB && TargetBB->getFragmentNum() != BB.getFragmentNum()) {
+ BitVector Available = Live;
+ Available.flip();
+ BitVector GPRegs(NumRegs, false);
+ MIB->getGPRegs(GPRegs, /*IncludeAlias=*/false);
+ Available &= GPRegs;
+ MIB->removeNonScavengeableRegs(Available);
+ const int Reg = Available.find_first();
+ if (Reg == -1) {
+ CanRelax = false;
+ break;
+ }
+ ScratchRegs[&Inst] = Reg;
+ }
+ Live = transfer(Inst, std::move(Live));
+ }
+ }
+ if (!CanRelax)
+ break;
+ }
+
+ // Unlike AArch64, RISC-V has no ABI-reserved linker scratch register. If
+ // every GPR is live across a cross-fragment edge, keep this function in a
+ // single fragment rather than silently clobbering program state.
+ if (!CanRelax) {
+ BC.errs() << "BOLT-WARNING: keeping " << BF
+ << " unsplit: no dead register for a RISC-V long jump\n";
+ BinaryFunction::BasicBlockOrderType Layout(BF.getLayout().block_begin(),
+ BF.getLayout().block_end());
+ for (BinaryBasicBlock &BB : BF)
+ BB.setFragmentNum(FragmentNum::main());
+ BF.getLayout().update(Layout);
+ BF.fixBranches();
+ return true;
+ }
+ }
+
auto isBranchOffsetInRange = [&](const MCInst &Inst, int64_t Offset) {
const unsigned Bits = MIB->getPCRelEncodingSize(Inst);
return isIntN(Bits, Offset);
};
+ // Output address ranges are persistent metadata used later for translating
+ // secondary entry points. Keep RISC-V's temporary relaxation offsets
+ // separate so fragment-relative offsets cannot leak into symbol rewriting.
+ DenseMap<const BinaryBasicBlock *, uint64_t> EstimatedStart;
+ DenseMap<const BinaryBasicBlock *, uint64_t> EstimatedEnd;
+ auto getEstimatedStart = [&](const BinaryBasicBlock *BB) {
+ return BC.isRISCV() ? EstimatedStart.lookup(BB)
+ : BB->getOutputStartAddress();
+ };
+ auto getEstimatedEnd = [&](const BinaryBasicBlock *BB) {
+ return BC.isRISCV() ? EstimatedEnd.lookup(BB) : BB->getOutputEndAddress();
+ };
+ auto setEstimatedRange = [&](BinaryBasicBlock *BB, uint64_t Start,
+ uint64_t End) {
+ if (BC.isRISCV()) {
+ EstimatedStart[BB] = Start;
+ EstimatedEnd[BB] = End;
+ } else {
+ BB->setOutputStartAddress(Start);
+ BB->setOutputEndAddress(End);
+ }
+ };
+
auto isBlockInRange = [&](const MCInst &Inst, uint64_t InstAddress,
const BinaryBasicBlock &BB) {
- const int64_t Offset = BB.getOutputStartAddress() - InstAddress;
+ const int64_t Offset = getEstimatedStart(&BB) - InstAddress;
return isBranchOffsetInRange(Inst, Offset);
};
@@ -703,20 +836,21 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
// Function fragments are relaxed independently.
for (FunctionFragment &FF : BF.getLayout().fragments()) {
- // Fill out code size estimation for the fragment. Use output BB address
- // ranges to store offsets from the start of the function fragment.
+ // Fill out code size estimation for the fragment.
uint64_t CodeSize = 0;
for (BinaryBasicBlock *BB : FF) {
- BB->setOutputStartAddress(CodeSize);
+ const uint64_t Start = CodeSize;
CodeSize += BB->estimateSize();
- BB->setOutputEndAddress(CodeSize);
+ setEstimatedRange(BB, Start, CodeSize);
}
// Dynamically-updated size of the fragment.
uint64_t FragmentSize = CodeSize;
- // Size of the trampoline in bytes.
- constexpr uint64_t TrampolineSize = 4;
+ // AArch64 trampolines start as one direct branch. RISC-V trampolines use
+ // AUIPC+JALR so that split fragments can be placed outside the +/-1 MiB
+ // JAL range.
+ const uint64_t TrampolineSize = BC.isRISCV() ? 8 : 4;
// Trampolines created for the fragment. DestinationBB -> TrampolineBB.
// NB: here we store only the first trampoline created for DestinationBB.
@@ -732,40 +866,39 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
auto addTrampolineAfter = [&](BinaryBasicBlock *BB,
const MCSymbol *TargetSym,
BinaryBasicBlock *TargetBB, uint64_t Count,
- uint64_t Offset = 0) {
+ int64_t BBSizeDelta = 0,
+ MCPhysReg ScratchReg = 0) {
FunctionTrampolines.emplace_back(BB ? BB : FF.back(),
BF.createBasicBlock());
BinaryBasicBlock *TrampolineBB = FunctionTrampolines.back().second.get();
- const uint64_t OldBBEnd = BB ? BB->getOutputEndAddress() : 0;
- if (BB && Offset)
- BB->setOutputEndAddress(OldBBEnd + Offset);
- Offset += TrampolineSize;
+ const uint64_t OldBBEnd = BB ? getEstimatedEnd(BB) : 0;
+ if (BB && BBSizeDelta)
+ setEstimatedRange(BB, getEstimatedStart(BB),
+ getEstimatedEnd(BB) + BBSizeDelta);
+ const int64_t Offset = BBSizeDelta + TrampolineSize;
- MCInst Inst;
+ InstructionListType Seq;
{
auto L = BC.scopeLock();
- MIB->createUncondBranch(Inst, TargetSym, BC.Ctx.get());
+ if (BC.isRISCV()) {
+ assert(TargetBB && "RISC-V trampoline requires a basic block target");
+ MIB->createLongJmp(Seq, TargetSym, BC.Ctx.get(),
+ /*IsTailCall=*/false, ScratchReg);
+ } else {
+ Seq.emplace_back();
+ MIB->createUncondBranch(Seq.back(), TargetSym, BC.Ctx.get());
+ }
}
- TrampolineBB->addInstruction(Inst);
+ TrampolineBB->addInstructions(Seq.begin(), Seq.end());
if (TargetBB)
TrampolineBB->addSuccessor(TargetBB, Count);
TrampolineBB->setExecutionCount(Count);
const uint64_t TrampolineAddress =
- BB ? BB->getOutputEndAddress() : FragmentSize;
- TrampolineBB->setOutputStartAddress(TrampolineAddress);
- TrampolineBB->setOutputEndAddress(TrampolineAddress + TrampolineSize);
+ BB ? getEstimatedEnd(BB) : FragmentSize;
+ setEstimatedRange(TrampolineBB, TrampolineAddress,
+ TrampolineAddress + TrampolineSize);
TrampolineBB->setFragmentNum(FF.getFragmentNum());
- // Shift the fragment-local output address range for blocks at or after
- // the old end address.
- auto adjustBasicBlockAddress = [](BinaryBasicBlock *BB, uint64_t Address,
- uint64_t Offset) {
- if (BB->getOutputStartAddress() < Address)
- return;
- BB->setOutputStartAddress(BB->getOutputStartAddress() + Offset);
- BB->setOutputEndAddress(BB->getOutputEndAddress() + Offset);
- };
-
if (TargetBB && !FragmentTrampolines.lookup(TargetBB))
FragmentTrampolines[TargetBB] = TrampolineBB;
@@ -780,8 +913,11 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
return TrampolineBB;
// Update offsets for blocks after BB.
- for (BinaryBasicBlock *IBB : FF)
- adjustBasicBlockAddress(IBB, OldBBEnd, Offset);
+ for (BinaryBasicBlock *IBB : FF) {
+ const uint64_t Start = getEstimatedStart(IBB);
+ if (Start >= OldBBEnd)
+ setEstimatedRange(IBB, Start + Offset, getEstimatedEnd(IBB) + Offset);
+ }
// Update offsets for trampolines in this fragment that are placed after
// the new trampoline. Note that trampoline blocks are not part of the
@@ -793,7 +929,9 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
continue;
if (IBB == TrampolineBB)
continue;
- adjustBasicBlockAddress(IBB, OldBBEnd, Offset);
+ const uint64_t Start = getEstimatedStart(IBB);
+ if (Start >= OldBBEnd)
+ setEstimatedRange(IBB, Start + Offset, getEstimatedEnd(IBB) + Offset);
}
return TrampolineBB;
@@ -810,15 +948,20 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
continue;
const MCSymbol *TargetSymbol = MIB->getTargetSymbol(*Inst);
- BB->eraseInstruction(BB->findInstruction(Inst));
-
BinaryBasicBlock::BinaryBranchInfo BI;
BinaryBasicBlock *TargetBB = BB->getSuccessor(TargetSymbol, BI);
+ if (!TargetBB || (BC.isRISCV() &&
+ TargetBB->getFragmentNum() == BB->getFragmentNum()))
+ continue;
+
+ const uint64_t BranchSize =
+ BC.isRISCV() ? BC.computeCodeSize(Inst, Inst + 1) : 4;
+ const MCPhysReg ScratchReg = ScratchRegs.lookup(Inst);
+ BB->eraseInstruction(BB->findInstruction(Inst));
- // Erasing the unconditional branch shrinks BB by one instruction.
BinaryBasicBlock *TrampolineBB =
addTrampolineAfter(BB, TargetBB->getLabel(), TargetBB, BI.Count,
- /*Offset=*/-4);
+ -static_cast<int64_t>(BranchSize), ScratchReg);
BB->replaceSuccessor(TargetBB, TrampolineBB, BI.Count);
}
}
@@ -837,7 +980,8 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
// Try to reuse an existing trampoline without introducing any new code.
BinaryBasicBlock *TrampolineBB = FragmentTrampolines.lookup(TargetBB);
- if (TrampolineBB && isBlockInRange(Inst, InstAddress, *TrampolineBB)) {
+ if (!BC.isRISCV() && TrampolineBB &&
+ isBlockInRange(Inst, InstAddress, *TrampolineBB)) {
BB->replaceSuccessor(TargetBB, TrampolineBB, Count);
TrampolineBB->setExecutionCount(TrampolineBB->getExecutionCount() +
Count);
@@ -850,10 +994,12 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
// of the fragment that is within the branch reach. Note that such
// trampoline may change address later and become unreachable in which
// case we will need further relaxation.
+ const MCPhysReg ScratchReg = ScratchRegs.lookup(&Inst);
const int64_t OffsetToEnd = FragmentSize - InstAddress;
if (Count == 0 && isBranchOffsetInRange(Inst, OffsetToEnd)) {
TrampolineBB =
- addTrampolineAfter(nullptr, TargetBB->getLabel(), TargetBB, Count);
+ addTrampolineAfter(nullptr, TargetBB->getLabel(), TargetBB, Count,
+ /*BBSizeDelta=*/0, ScratchReg);
BB->replaceSuccessor(TargetBB, TrampolineBB, Count);
auto L = BC.scopeLock();
MIB->replaceBranchTarget(Inst, TrampolineBB->getLabel(), BC.Ctx.get());
@@ -873,7 +1019,8 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
if (ShouldReverseBranch && !IsReversibleBranch) {
const uint64_t NextCount = BB->getBranchInfo(*NextBB).Count;
BinaryBasicBlock *FallThrough =
- addTrampolineAfter(BB, NextBB->getLabel(), NextBB, NextCount);
+ addTrampolineAfter(BB, NextBB->getLabel(), NextBB, NextCount,
+ /*BBSizeDelta=*/0, ScratchReg);
BB->replaceSuccessor(NextBB, FallThrough, NextCount);
}
@@ -891,12 +1038,15 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
const uint64_t NewBBSize = BB->estimateSize();
// Create a trampoline basic block for the original taken target.
- TrampolineBB = addTrampolineAfter(BB, TargetBB->getLabel(), TargetBB,
- Count, NewBBSize - OldBBSize);
+ TrampolineBB = addTrampolineAfter(
+ BB, TargetBB->getLabel(), TargetBB, Count,
+ static_cast<int64_t>(NewBBSize) - static_cast<int64_t>(OldBBSize),
+ ScratchReg);
} else {
// Create a trampoline basic block for the taken target of the branch.
TrampolineBB =
- addTrampolineAfter(BB, TargetBB->getLabel(), TargetBB, Count);
+ addTrampolineAfter(BB, TargetBB->getLabel(), TargetBB, Count,
+ /*BBSizeDelta=*/0, ScratchReg);
auto L = BC.scopeLock();
MIB->replaceBranchTarget(Inst, TrampolineBB->getLabel(), BC.Ctx.get());
}
@@ -914,31 +1064,39 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
++NumIterations;
for (auto BBI = FF.begin(); BBI != FF.end(); ++BBI) {
BinaryBasicBlock *BB = *BBI;
- uint64_t NextInstOffset = BB->getOutputStartAddress();
+ uint64_t NextInstOffset = getEstimatedStart(BB);
// Branch reversal may replace the current instruction with a sequence.
// Use an index so the next instruction is reloaded after the mutation.
for (size_t I = 0; I < BB->size(); ++I) {
MCInst &Inst = *(BB->begin() + I);
const size_t InstAddress = NextInstOffset;
if (!MIB->isPseudo(Inst))
- NextInstOffset += 4;
+ NextInstOffset +=
+ BC.isRISCV() ? BC.computeCodeSize(&Inst, &Inst + 1) : 4;
if (!mayNeedStub(BF.getBinaryContext(), Inst))
continue;
+ const MCSymbol *TargetSymbol = MIB->getTargetSymbol(Inst);
const size_t BitsAvailable = MIB->getPCRelEncodingSize(Inst);
- // Span of +/-128MB.
- if (BitsAvailable == LongestJumpBits)
+ // AArch64 compact code model keeps fragments within the range of B.
+ if (!BC.isRISCV() && BitsAvailable == LongestJumpBits)
continue;
- const MCSymbol *TargetSymbol = MIB->getTargetSymbol(Inst);
-
if (BF.isSimple()) {
BinaryBasicBlock *TargetBB = BB->getSuccessor(TargetSymbol);
assert(TargetBB &&
"Basic block target expected for conditional branch.");
+ // Existing intra-fragment RISC-V branches are handled by JITLink's
+ // normal branch relaxation. This pass is responsible for edges
+ // that become unrepresentable specifically because of a function
+ // split.
+ if (BC.isRISCV() &&
+ TargetBB->getFragmentNum() == FF.getFragmentNum())
+ continue;
+
// Check if the relaxation is needed.
if (TargetBB->getFragmentNum() == FF.getFragmentNum() &&
isBlockInRange(Inst, InstAddress, *TargetBB))
@@ -1322,6 +1480,15 @@ void LongJmpPass::relaxCalls(BinaryContext &BC) {
}
Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
+ if (BC.isRISCV()) {
+ BC.outs() << "BOLT-INFO: relaxing RISC-V cross-fragment branches\n";
+ for (BinaryFunction *BF : BC.getOutputBinaryFunctions()) {
+ if (!BC.shouldEmit(*BF) || !BF->isSimple() || !BF->isSplit())
+ continue;
+ relaxLocalBranches(*BF);
+ }
+ return Error::success();
+ }
assert((opts::CompactCodeModel || opts::ExperimentalRelaxation ||
opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) &&
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index e61297f94ff98..8e1a4aee8e2fa 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -539,12 +539,16 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
if (BC.isAArch64()) {
Manager.registerPass(
std::make_unique<AArch64RelaxationPass>(PrintAArch64Relaxation));
+ }
+ if (BC.isAArch64() || BC.isRISCV()) {
// Tighten branches according to offset differences between branch and
// targets. No extra instructions after this pass, otherwise we may have
// relocations out of range and crash during linking.
Manager.registerPass(std::make_unique<LongJmpPass>(PrintLongJmp));
+ }
+ if (BC.isAArch64()) {
Manager.registerPass(
std::make_unique<PointerAuthCFIFixup>(PrintPAuthCFIFixup));
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 67a19dcc47124..b06baca936150 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -443,6 +443,12 @@ RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,
return;
} else {
Features.reset(new SubtargetFeatures(*FeaturesOrErr));
+ // EF_RISCV_RVE selects the E ABI even when the input has no
+ // .riscv.attributes architecture string. ObjectFile::getFeatures()
+ // currently derives RVC from e_flags but not RVE, so preserve this ABI
+ // constraint explicitly for register analysis and code generation.
+ if (File->getPlatformFlags() & ELF::EF_RISCV_RVE)
+ Features->AddFeature("e");
}
}
@@ -5671,6 +5677,17 @@ void RewriteInstance::updateELFSymbolTable(
Expected<StringRef> SymbolName = Symbol.getName(StringSection);
assert(SymbolName && "cannot get symbol name");
+ // Mapping symbols can share the exact address of a function entry, but
+ // they are code/data metadata rather than function aliases. Let the
+ // marker-specific path below handle them; otherwise addExtraSymbols()
+ // creates invalid split names such as "$xrv64i...cold.0".
+ auto IsMarkerSymbol = [&]() {
+ return BC->getMarkerType(Symbol.getType(), Symbol.st_size,
+ *SymbolName) != MarkerSymType::NONE;
+ };
+ if (Function && IsMarkerSymbol())
+ Function = nullptr;
+
auto updateSymbolValue = [&](const StringRef Name,
std::optional<uint64_t> Value = std::nullopt) {
NewSymbol.st_value = Value ? *Value : getNewValueForSymbol(Name);
@@ -5722,10 +5739,6 @@ void RewriteInstance::updateELFSymbolTable(
// update their addresses to reflect the output layout.
// Skip AArch64/RISC-V marker symbols ($d, $x) inside functions —
// BOLT generates its own via addExtraSymbols.
- auto IsMarkerSymbol = [&]() {
- return BC->getMarkerType(Symbol.getType(), Symbol.st_size,
- *SymbolName) != MarkerSymType::NONE;
- };
const bool IsLocalLabel = Symbol.getType() == ELF::STT_NOTYPE &&
Symbol.getBinding() == ELF::STB_LOCAL &&
Symbol.st_size == 0 && !IsMarkerSymbol();
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 8d7fee733b9c3..5629b558a2da3 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -2771,7 +2771,9 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
}
void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
- MCContext *Ctx, bool IsTailCall) override {
+ MCContext *Ctx, bool IsTailCall,
+ MCPhysReg ScratchReg) override {
+ (void)ScratchReg;
// ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
// Standard for the ARM 64-bit Architecture (AArch64)".
// The sequence of instructions we create here is the following:
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index eed94ff5e307d..23eeafd259711 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -31,6 +31,7 @@ namespace {
class RISCVMCPlusBuilder : public MCPlusBuilder {
bool isRV64() const { return STI->hasFeature(RISCV::Feature64Bit); }
+ bool isRVE() const { return STI->hasFeature(RISCV::FeatureStdExtE); }
unsigned regSize() const { return isRV64() ? 8 : 4; }
unsigned loadOpc() const { return isRV64() ? RISCV::LD : RISCV::LW; }
unsigned storeOpc() const { return isRV64() ? RISCV::SD : RISCV::SW; }
@@ -41,6 +42,14 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
public:
using MCPlusBuilder::MCPlusBuilder;
+ BitVector getRegsUsedAsParams() const override {
+ BitVector Regs(RegInfo->getNumRegs(), false);
+ const MCPhysReg LastArgReg = isRVE() ? RISCV::X15 : RISCV::X17;
+ for (MCPhysReg Reg = RISCV::X10; Reg <= LastArgReg; ++Reg)
+ Regs |= getAliases(Reg);
+ return Regs;
+ }
+
std::unique_ptr<MCSymbolizer>
createTargetSymbolizer(BinaryFunction &Function,
bool CreateNewSymbols) const override {
@@ -62,6 +71,8 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
Regs |= getAliases(RISCV::X2);
Regs |= getAliases(RISCV::X8);
Regs |= getAliases(RISCV::X9);
+ if (isRVE())
+ return;
Regs |= getAliases(RISCV::X18);
Regs |= getAliases(RISCV::X19);
Regs |= getAliases(RISCV::X20);
@@ -74,6 +85,32 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
Regs |= getAliases(RISCV::X27);
}
+ void getDefaultLiveOut(BitVector &Regs) const override {
+ Regs |= getAliases(RISCV::X10);
+ Regs |= getAliases(RISCV::X11);
+ }
+
+ void getGPRegs(BitVector &Regs, bool IncludeAlias = true) const override {
+ const MCPhysReg LastGPR = isRVE() ? RISCV::X15 : RISCV::X31;
+ for (MCPhysReg Reg = RISCV::X1; Reg <= LastGPR; ++Reg) {
+ if (IncludeAlias)
+ Regs |= getAliases(Reg);
+ else
+ Regs.set(Reg);
+ }
+ }
+
+ void removeNonScavengeableRegs(BitVector &Regs) const override {
+ BitVector ExclusionMask(RegInfo->getNumRegs(), false);
+ ExclusionMask |= getAliases(RISCV::X1); // return address
+ ExclusionMask |= getAliases(RISCV::X2); // stack pointer
+ ExclusionMask |= getAliases(RISCV::X3); // global pointer
+ ExclusionMask |= getAliases(RISCV::X4); // thread pointer
+ ExclusionMask |= getAliases(RISCV::X8); // frame pointer
+ ExclusionMask.flip();
+ Regs &= ExclusionMask;
+ }
+
bool shouldRecordCodeRelocation(uint32_t RelType) const override {
switch (RelType) {
case ELF::R_RISCV_JAL:
@@ -179,6 +216,29 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
return {Inst};
}
+ int getPCRelEncodingSize(const MCInst &Inst) const override {
+ switch (Inst.getOpcode()) {
+ default:
+ llvm_unreachable("Failed to get RISC-V PC-relative encoding size");
+ case RISCV::C_BEQZ:
+ case RISCV::C_BNEZ:
+ return 9;
+ case RISCV::C_J:
+ return 12;
+ case RISCV::BEQ:
+ case RISCV::BNE:
+ case RISCV::BLT:
+ case RISCV::BGE:
+ case RISCV::BLTU:
+ case RISCV::BGEU:
+ return 13;
+ case RISCV::JAL:
+ return 21;
+ }
+ }
+
+ int getUncondBranchEncodingSize() const override { return 21; }
+
void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
MCContext *Ctx) const override {
assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
@@ -622,6 +682,29 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
Seq.swap(Insts);
}
+ void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
+ MCContext *Ctx, bool IsTailCall,
+ MCPhysReg ScratchReg) override {
+ assert(ScratchReg && "RISC-V long jump requires a scratch register");
+ MCSymbol *AuipcLabel = Ctx->createNamedTempSymbol("long_jmp");
+
+ MCInst Inst = MCInstBuilder(RISCV::AUIPC).addReg(ScratchReg).addImm(0);
+ setOperandToSymbolRef(Inst, /*OpNum=*/1, Target, /*Addend=*/0, Ctx,
+ ELF::R_RISCV_PCREL_HI20);
+ setInstLabel(Inst, AuipcLabel);
+ Seq.emplace_back(std::move(Inst));
+
+ Inst = MCInstBuilder(RISCV::JALR)
+ .addReg(RISCV::X0)
+ .addReg(ScratchReg)
+ .addImm(0);
+ setOperandToSymbolRef(Inst, /*OpNum=*/2, AuipcLabel, /*Addend=*/0, Ctx,
+ ELF::R_RISCV_PCREL_LO12_I);
+ if (IsTailCall)
+ setTailCall(Inst);
+ Seq.emplace_back(std::move(Inst));
+ }
+
InstructionListType createGetter(MCContext *Ctx, const char *name) const {
InstructionListType Insts(4);
MCSymbol *Locs = Ctx->getOrCreateSymbol(name);
diff --git a/bolt/test/RISCV/split-functions-indirect-call-scratch.s b/bolt/test/RISCV/split-functions-indirect-call-scratch.s
new file mode 100644
index 0000000000000..a87027571ef53
--- /dev/null
+++ b/bolt/test/RISCV/split-functions-indirect-call-scratch.s
@@ -0,0 +1,43 @@
+## Check that register scavenging for a cross-fragment long jump accounts for
+## the explicit target register of an indirect call in the destination block.
+## The trampoline must not clobber t0 before the cold block calls through it.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1
+# RUN: llvm-objdump -d --no-show-raw-insn %t.bolt | FileCheck %s
+
+# CHECK-LABEL: <_start>:
+# CHECK: auipc t0,
+# CHECK-NEXT: addi t0, t0,
+# CHECK: auipc t1,
+# CHECK-NEXT: {{(jalr zero,|jr)}} {{.*}}(t1)
+# CHECK: auipc t1,
+# CHECK-NEXT: {{(jalr zero,|jr)}} {{.*}}(t1)
+# CHECK-LABEL: <_start.cold.0>:
+# CHECK: jalr t0
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+1:
+ auipc t0, %pcrel_hi(callee)
+ addi t0, t0, %pcrel_lo(1b)
+ beq a0, zero, .Lcold
+ li a0, 1
+ ret
+.Lcold:
+ jalr ra, t0, 0
+ ret
+ .size _start, .-_start
+
+ .globl callee
+ .type callee, @function
+callee:
+ li a0, 0
+ ret
+ .size callee, .-callee
+
+ .reloc 0, R_RISCV_NONE
diff --git a/bolt/test/RISCV/split-functions-long-jump.s b/bolt/test/RISCV/split-functions-long-jump.s
new file mode 100644
index 0000000000000..9cf576d2f8038
--- /dev/null
+++ b/bolt/test/RISCV/split-functions-long-jump.s
@@ -0,0 +1,48 @@
+## Check that a branch crossing a split-function fragment is redirected through
+## a local trampoline. The trampoline uses AUIPC+JALR instead of JAL so the
+## cold fragment can be placed outside the +/-1 MiB JAL range.
+
+# RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1
+# RUN: llvm-objdump -d %t.bolt | FileCheck %s
+# RUN: llvm-readelf -s %t.bolt | FileCheck --check-prefix=SYMBOLS %s
+# RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj -o %t.32.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.32.exe %t.32.o
+# RUN: llvm-bolt %t.32.exe -o %t.32.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1
+# RUN: llvm-objdump -d %t.32.bolt | FileCheck %s
+# RUN: llvm-readelf -s %t.32.bolt | FileCheck --check-prefix=SYMBOLS %s
+# RUN: llvm-mc -triple riscv32 -mattr=+e -filetype=obj -o %t.e.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.e.exe %t.e.o
+# RUN: llvm-bolt %t.e.exe -o %t.e.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1
+# RUN: llvm-objdump -d %t.e.bolt 2>&1 | FileCheck %s
+# RUN: llvm-readelf -s %t.e.bolt | FileCheck --check-prefix=SYMBOLS %s
+
+# CHECK: Disassembly of section .text:
+# CHECK-LABEL: <_start>:
+# CHECK: auipc [[REG:[a-z0-9]+]],
+# CHECK-NEXT: {{(jalr zero,|jr)}} {{.*}}([[REG]])
+# CHECK: Disassembly of section .text.cold:
+# CHECK-LABEL: <secondary>:
+# SYMBOLS: FUNC GLOBAL DEFAULT {{[0-9]+}} secondary
+# SYMBOLS-NOT: $x{{.*}}.cold
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ beq a0, zero, .Lcold
+ .globl secondary
+ .type secondary, @function
+secondary:
+ li a0, 1
+ ret
+.Lcold:
+ li a0, 2
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
diff --git a/bolt/test/RISCV/split-functions-no-scratch-rve.s b/bolt/test/RISCV/split-functions-no-scratch-rve.s
new file mode 100644
index 0000000000000..3cf385916ad53
--- /dev/null
+++ b/bolt/test/RISCV/split-functions-no-scratch-rve.s
@@ -0,0 +1,33 @@
+## RVE only has x0-x15. If every usable RVE GPR is live across a split edge,
+## check that BOLT does not select an unavailable x16-x31 register for the
+## AUIPC+JALR trampoline and keeps the function unsplit instead.
+
+# RUN: llvm-mc -triple riscv32 -mattr=+e -filetype=obj -o %t.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.bolt | FileCheck --check-prefix=SECTIONS %s
+
+# CHECK: BOLT-WARNING: keeping _start unsplit: no dead register for a RISC-V long jump
+# SECTIONS-NOT: .text.cold
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ beq a0, zero, .Lcold
+ ret
+.Lcold:
+ add a0, a0, t0
+ add a0, a0, t1
+ add a0, a0, t2
+ add a0, a0, s1
+ add a0, a0, a1
+ add a0, a0, a2
+ add a0, a0, a3
+ add a0, a0, a4
+ add a0, a0, a5
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
diff --git a/bolt/test/RISCV/split-functions-no-scratch.s b/bolt/test/RISCV/split-functions-no-scratch.s
new file mode 100644
index 0000000000000..b3841268a318c
--- /dev/null
+++ b/bolt/test/RISCV/split-functions-no-scratch.s
@@ -0,0 +1,49 @@
+## RISC-V has no ABI-reserved linker scratch register. If every usable GPR is
+## live across a split edge, check that BOLT keeps that function unsplit rather
+## than clobbering state in an AUIPC+JALR trampoline.
+
+# RUN: llvm-mc -triple riscv64 -filetype=obj -o %t.o %s
+# RUN: ld.lld --emit-relocs -e _start -o %t.exe %t.o
+# RUN: llvm-bolt %t.exe -o %t.bolt -split-functions \
+# RUN: -split-strategy=random2 -bolt-seed=1 2>&1 | FileCheck %s
+# RUN: llvm-readelf -S %t.bolt | FileCheck --check-prefix=SECTIONS %s
+
+# CHECK: BOLT-WARNING: keeping _start unsplit: no dead register for a RISC-V long jump
+# SECTIONS-NOT: .text.cold
+
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ beq a0, zero, .Lcold
+ ret
+.Lcold:
+ add a0, a0, t0
+ add a0, a0, t1
+ add a0, a0, t2
+ add a0, a0, s1
+ add a0, a0, a1
+ add a0, a0, a2
+ add a0, a0, a3
+ add a0, a0, a4
+ add a0, a0, a5
+ add a0, a0, a6
+ add a0, a0, a7
+ add a0, a0, s2
+ add a0, a0, s3
+ add a0, a0, s4
+ add a0, a0, s5
+ add a0, a0, s6
+ add a0, a0, s7
+ add a0, a0, s8
+ add a0, a0, s9
+ add a0, a0, s10
+ add a0, a0, s11
+ add a0, a0, t3
+ add a0, a0, t4
+ add a0, a0, t5
+ add a0, a0, t6
+ ret
+ .size _start, .-_start
+
+ .reloc 0, R_RISCV_NONE
>From 50b3b5100890081304db3b96879782c6134f6492 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Wed, 29 Jul 2026 20:17:18 +0800
Subject: [PATCH 06/10] [BOLT][RISCV] Implement indirect PLT calls
---
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 53 ++++++++++++++++++++
bolt/test/RISCV/plt-call.test | 44 ++++++++++++++++
2 files changed, 97 insertions(+)
create mode 100644 bolt/test/RISCV/plt-call.test
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 23eeafd259711..2306b462bc4a0 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -350,6 +350,59 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
return createCall(RISCV::PseudoTAIL, Inst, Target, Ctx);
}
+ InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
+ const MCSymbol *TargetLocation,
+ MCContext *Ctx) override {
+ const bool IsTailCall = isTailCall(DirectCall);
+ assert(((DirectCall.getOpcode() == RISCV::PseudoCALL && !IsTailCall) ||
+ (DirectCall.getOpcode() == RISCV::PseudoTAIL && IsTailCall)) &&
+ "RISC-V direct (tail) call instruction expected");
+
+ // Load the resolved function address directly from its GOT slot:
+ //
+ // auipc t3, %pcrel_hi(TargetLocation)
+ // l[dw] t3, %pcrel_lo(.Lpcrel_hi)(t3)
+ // jalr ra, t3, 0
+ //
+ // A tail call uses zero instead of ra as the JALR destination.
+ InstructionListType Code;
+ // Use t3 (x28), the scratch register used by linker-generated RISC-V
+ // PLT/IPLT entries. It is caller-saved, is not an argument register, and
+ // the original call through the PLT already clobbers it.
+ const MCPhysReg PLTScratchReg = RISCV::X28;
+ MCSymbol *AUIPCLabel = Ctx->createNamedTempSymbol("pcrel_hi");
+
+ MCInst InstAUIPC =
+ MCInstBuilder(RISCV::AUIPC).addReg(PLTScratchReg).addImm(0);
+ // TargetLocation is already registered at the existing GOT slot, so use a
+ // direct PC-relative relocation to that slot instead of R_RISCV_GOT_HI20,
+ // which is used when starting from the referenced function symbol.
+ setOperandToSymbolRef(InstAUIPC, /*OpNum=*/1, TargetLocation,
+ /*Addend=*/0, Ctx, ELF::R_RISCV_PCREL_HI20);
+ setInstLabel(InstAUIPC, AUIPCLabel);
+ Code.emplace_back(std::move(InstAUIPC));
+
+ // Load the call target from the GOT slot using LD on RV64 or LW on RV32.
+ MCInst InstLoad = MCInstBuilder(loadOpc())
+ .addReg(PLTScratchReg)
+ .addReg(PLTScratchReg)
+ .addImm(0);
+ // Pair the I-type LD/LW immediate with the label on AUIPC. RISC-V
+ // R_RISCV_PCREL_LO12_I relocations name the corresponding HI20 location.
+ setOperandToSymbolRef(InstLoad, /*OpNum=*/2, AUIPCLabel,
+ /*Addend=*/0, Ctx, ELF::R_RISCV_PCREL_LO12_I);
+ Code.emplace_back(std::move(InstLoad));
+
+ MCInst InstCall = MCInstBuilder(RISCV::JALR)
+ .addReg(IsTailCall ? RISCV::X0 : RISCV::X1)
+ .addReg(PLTScratchReg)
+ .addImm(0);
+ moveAnnotations(std::move(DirectCall), InstCall);
+ Code.emplace_back(std::move(InstCall));
+
+ return Code;
+ }
+
bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
const MCSymbol *&TBB, const MCSymbol *&FBB,
MCInst *&CondBranch,
diff --git a/bolt/test/RISCV/plt-call.test b/bolt/test/RISCV/plt-call.test
new file mode 100644
index 0000000000000..76f850e892e59
--- /dev/null
+++ b/bolt/test/RISCV/plt-call.test
@@ -0,0 +1,44 @@
+// Verify that PLTCall optimization works on RISC-V.
+
+// RUN: split-file %s %t.dir
+// RUN: llvm-mc -triple=riscv64 -filetype=obj -o %t.dir/main.o %t.dir/main.s
+// RUN: llvm-mc -triple=riscv64 -filetype=obj -o %t.dir/lib.o %t.dir/lib.s
+// RUN: ld.lld -shared -soname libplt.so -o %t.dir/libplt.so %t.dir/lib.o
+// RUN: ld.lld --no-pie --emit-relocs -z now \
+// RUN: -dynamic-linker /lib/ld.so.1 %t.dir/main.o %t.dir/libplt.so \
+// RUN: -o %t.exe
+// RUN: llvm-bolt %t.exe -o %t.bolt --plt=all --print-plt \
+// RUN: --print-only=_start | FileCheck %s
+
+// Call to foo.
+// CHECK: auipc t3, %pcrel_hi(foo at GOT)
+// CHECK-NEXT: ld t3, %pcrel_lo({{.*}})(t3)
+// CHECK-NEXT: jalr t3 # PLTCall: 1
+
+// Tail call to bar.
+// CHECK: auipc t3, %pcrel_hi(bar at GOT)
+// CHECK-NEXT: ld t3, %pcrel_lo({{.*}})(t3)
+// CHECK-NEXT: jr t3 # TAILCALL # PLTCall: 1
+
+//--- main.s
+ .text
+ .globl _start
+ .type _start, @function
+_start:
+ call foo
+ tail bar
+ .size _start, .-_start
+
+//--- lib.s
+ .text
+ .globl foo
+ .type foo, @function
+foo:
+ ret
+ .size foo, .-foo
+
+ .globl bar
+ .type bar, @function
+bar:
+ ret
+ .size bar, .-bar
>From 8ac4553cd712a70c3e5656eded4bc4a2a9a5180c Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Mon, 3 Aug 2026 15:58:27 +0800
Subject: [PATCH 07/10] [BOLT] Track jump-table entry size and signedness
---
bolt/include/bolt/Core/BinaryContext.h | 16 +++--
bolt/include/bolt/Core/JumpTable.h | 6 +-
bolt/include/bolt/Core/MCPlusBuilder.h | 6 +-
bolt/include/bolt/Core/Relocation.h | 3 +
bolt/lib/Core/BinaryContext.cpp | 66 +++++++++++++------
bolt/lib/Core/BinaryFunction.cpp | 44 +++++++++++--
bolt/lib/Core/JumpTable.cpp | 15 +++--
bolt/lib/Core/Relocation.cpp | 14 ++++
bolt/lib/Passes/IndirectCallPromotion.cpp | 6 +-
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 17 +++--
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 5 +-
bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 15 +++--
12 files changed, 154 insertions(+), 59 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 92cd853870cae..283ef5a74d96e 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -642,7 +642,9 @@ class BinaryContext {
/// element of the pair.
const MCSymbol *getOrCreateJumpTable(BinaryFunction &Function,
uint64_t Address,
- JumpTable::JumpTableType Type);
+ JumpTable::JumpTableType Type,
+ uint64_t EntrySize = 0,
+ bool EntriesAreSigned = false);
/// Analyze a possible jump table of type \p Type at a given \p Address.
/// \p BF is a function referencing the jump table.
@@ -654,12 +656,12 @@ class BinaryContext {
///
/// Optionally, populate \p Address from jump table entries. The entries
/// could be partially populated if the jump table detection fails.
- bool analyzeJumpTable(const uint64_t Address,
- const JumpTable::JumpTableType Type,
- const BinaryFunction &BF,
- const uint64_t NextJTAddress = 0,
- JumpTable::AddressesType *EntriesAsAddress = nullptr,
- bool *HasEntryInFragment = nullptr) const;
+ bool
+ analyzeJumpTable(const uint64_t Address, const JumpTable::JumpTableType Type,
+ const BinaryFunction &BF, const uint64_t NextJTAddress = 0,
+ JumpTable::AddressesType *EntriesAsAddress = nullptr,
+ bool *HasEntryInFragment = nullptr, uint64_t EntrySize = 0,
+ bool EntriesAreSigned = false) const;
/// After jump table locations are established, this function will populate
/// their EntriesAsAddress based on memory contents.
diff --git a/bolt/include/bolt/Core/JumpTable.h b/bolt/include/bolt/Core/JumpTable.h
index 52b9ccee1f7e1..0f44252dd47e3 100644
--- a/bolt/include/bolt/Core/JumpTable.h
+++ b/bolt/include/bolt/Core/JumpTable.h
@@ -66,6 +66,9 @@ class JumpTable : public BinaryData {
/// The type of this jump table.
JumpTableType Type;
+ /// Whether entries are sign-extended when loaded by the dispatch sequence.
+ bool EntriesAreSigned;
+
/// Whether this jump table has entries pointing to multiple functions.
bool IsSplit{false};
@@ -95,7 +98,8 @@ class JumpTable : public BinaryData {
private:
/// Constructor should only be called by a BinaryContext.
JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize,
- JumpTableType Type, LabelMapType &&Labels, BinarySection &Section);
+ bool EntriesAreSigned, JumpTableType Type, LabelMapType &&Labels,
+ BinarySection &Section);
public:
/// Return the size of the jump table.
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 9882980396b50..1e0bdcd54117c 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1774,11 +1774,15 @@ class MCPlusBuilder {
/// will be set to the different components of the branch. \p MemLocInstr
/// is the instruction that loads up the indirect function pointer. It may
/// or may not be same as \p Instruction.
+ /// \p EntrySize and \p EntrySigned describe the jump-table entry loaded by
+ /// the matched instruction sequence. A zero entry size requests the target's
+ /// default for the detected jump-table type.
virtual IndirectBranchType analyzeIndirectBranch(
MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
const unsigned PtrSize, MCInst *&MemLocInstr, unsigned &BaseRegNum,
unsigned &IndexRegNum, int64_t &DispValue, const MCExpr *&DispExpr,
- MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInst) const {
+ uint64_t &EntrySize, bool &EntrySigned, MCInst *&PCRelBaseOut,
+ MCInst *&FixedEntryLoadInst) const {
llvm_unreachable("not implemented");
return IndirectBranchType::UNKNOWN;
}
diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h
index 8ee3e2587cb9c..d3f232055f8c8 100644
--- a/bolt/include/bolt/Core/Relocation.h
+++ b/bolt/include/bolt/Core/Relocation.h
@@ -150,6 +150,9 @@ class Relocation {
/// Return code for a PC-relative 8-byte relocation
static uint32_t getPC64();
+ /// Return code for an ABS 4-byte relocation
+ static uint32_t getAbs32();
+
/// Return code for a ABS 8-byte relocation
static uint32_t getAbs64();
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 2877a795d7bd2..920e0c6c796d9 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -630,12 +630,11 @@ MemoryContentsType BinaryContext::analyzeMemoryAt(uint64_t Address,
return MemoryContentsType::UNKNOWN;
}
-bool BinaryContext::analyzeJumpTable(const uint64_t Address,
- const JumpTable::JumpTableType Type,
- const BinaryFunction &BF,
- const uint64_t NextJTAddress,
- JumpTable::AddressesType *EntriesAsAddress,
- bool *HasEntryInFragment) const {
+bool BinaryContext::analyzeJumpTable(
+ const uint64_t Address, const JumpTable::JumpTableType Type,
+ const BinaryFunction &BF, const uint64_t NextJTAddress,
+ JumpTable::AddressesType *EntriesAsAddress, bool *HasEntryInFragment,
+ uint64_t EntrySize, bool EntriesAreSigned) const {
// Target address of __builtin_unreachable.
const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();
@@ -696,7 +695,11 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
Address, BF.getPrintName(),
Type == JTT::JTT_PIC ? "PIC" : "Normal");
});
- const uint64_t EntrySize = getJumpTableEntrySize(Type);
+ if (!EntrySize)
+ EntrySize = getJumpTableEntrySize(Type);
+ EntriesAreSigned |= Type == JumpTable::JTT_PIC;
+ if (UpperBound < Address || UpperBound - Address < EntrySize)
+ return false;
for (uint64_t EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;
EntryAddress += EntrySize) {
LLVM_DEBUG(dbgs() << " * Checking 0x" << Twine::utohexstr(EntryAddress)
@@ -717,10 +720,22 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
}
}
- const uint64_t Value =
- (Type == JumpTable::JTT_PIC)
- ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize)
- : *getPointerAtAddress(EntryAddress);
+ uint64_t Value;
+ if (EntriesAreSigned) {
+ ErrorOr<int64_t> SignedValue =
+ getSignedValueAtAddress(EntryAddress, EntrySize);
+ if (!SignedValue)
+ break;
+ Value = static_cast<uint64_t>(*SignedValue);
+ if (Type == JumpTable::JTT_PIC)
+ Value += Address;
+ } else {
+ ErrorOr<uint64_t> UnsignedValue =
+ getUnsignedValueAtAddress(EntryAddress, EntrySize);
+ if (!UnsignedValue)
+ break;
+ Value = *UnsignedValue;
+ }
// __builtin_unreachable() case.
if (Value == UnreachableAddress) {
@@ -792,7 +807,8 @@ void BinaryContext::populateJumpTables() {
const bool Success =
analyzeJumpTable(JT->getAddress(), JT->Type, *(JT->Parents[0]),
- NextJTAddress, &JT->EntriesAsAddress, &JT->IsSplit);
+ NextJTAddress, &JT->EntriesAsAddress, &JT->IsSplit,
+ JT->EntrySize, JT->EntriesAreSigned);
if (!Success) {
// Re-analysis here is stricter than during disassembly (the referenced
// function is now disassembled), so it may fail on a table we accepted
@@ -836,7 +852,7 @@ void BinaryContext::populateJumpTables() {
for (uint64_t Address = JT->getAddress();
Address < JT->getAddress() + JT->getSize();
Address += JT->EntrySize) {
- DataPCRelocations.erase(DataPCRelocations.find(Address));
+ DataPCRelocations.erase(Address);
}
}
@@ -926,11 +942,18 @@ BinaryFunction *BinaryContext::createBinaryFunction(
const MCSymbol *
BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
- JumpTable::JumpTableType Type) {
+ JumpTable::JumpTableType Type,
+ uint64_t EntrySize, bool EntriesAreSigned) {
+ EntriesAreSigned |= Type == JumpTable::JTT_PIC;
+
// Two fragments of same function access same jump table
if (JumpTable *JT = getJumpTableContainingAddress(Address)) {
assert(JT->Type == Type && "jump table types have to match");
assert(Address == JT->getAddress() && "unexpected non-empty jump table");
+ assert((!EntrySize || JT->EntrySize == EntrySize) &&
+ "jump table entry sizes have to match");
+ assert((!EntrySize || JT->EntriesAreSigned == EntriesAreSigned) &&
+ "jump table entry signedness has to match");
if (llvm::is_contained(JT->Parents, &Function))
return JT->getFirstLabel();
@@ -962,7 +985,8 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
JTLabel = Object->getSymbol();
}
- const uint64_t EntrySize = getJumpTableEntrySize(Type);
+ if (!EntrySize)
+ EntrySize = getJumpTableEntrySize(Type);
if (!JTLabel) {
const std::string JumpTableName = generateJumpTableName(Function, Address);
JTLabel = registerNameAtAddress(JumpTableName, Address, 0, EntrySize);
@@ -971,8 +995,8 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: creating jump table " << JTLabel->getName()
<< " in function " << Function << '\n');
- JumpTable *JT = new JumpTable(*JTLabel, Address, EntrySize, Type,
- JumpTable::LabelMapType{{0, JTLabel}},
+ JumpTable *JT = new JumpTable(*JTLabel, Address, EntrySize, EntriesAreSigned,
+ Type, JumpTable::LabelMapType{{0, JTLabel}},
*getSectionForAddress(Address));
JT->Parents.push_back(&Function);
if (opts::Verbosity > 2)
@@ -1000,10 +1024,10 @@ BinaryContext::duplicateJumpTable(BinaryFunction &Function, JumpTable *JT,
assert(Found && "Label not found");
(void)Found;
MCSymbol *NewLabel = Ctx->createNamedTempSymbol("duplicatedJT");
- JumpTable *NewJT =
- new JumpTable(*NewLabel, JT->getAddress(), JT->EntrySize, JT->Type,
- JumpTable::LabelMapType{{Offset, NewLabel}},
- *getSectionForAddress(JT->getAddress()));
+ JumpTable *NewJT = new JumpTable(*NewLabel, JT->getAddress(), JT->EntrySize,
+ JT->EntriesAreSigned, JT->Type,
+ JumpTable::LabelMapType{{Offset, NewLabel}},
+ *getSectionForAddress(JT->getAddress()));
NewJT->Parents = JT->Parents;
NewJT->Entries = JT->Entries;
NewJT->Counts = JT->Counts;
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 6c5e833539f90..43a128dc17850 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -825,6 +825,8 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
unsigned BaseRegNum, IndexRegNum;
int64_t DispValue;
const MCExpr *DispExpr;
+ uint64_t EntrySize;
+ bool EntrySigned;
// In AArch, identify the instruction adding the PC-relative offset to
// jump table entries to correctly decode it.
@@ -848,12 +850,13 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
IndirectBranchType BranchType = BC.MIB->analyzeIndirectBranch(
Instruction, Begin, Instructions.end(), PtrSize, MemLocInstr, BaseRegNum,
- IndexRegNum, DispValue, DispExpr, PCRelBaseInstr, FixedEntryLoadInstr);
+ IndexRegNum, DispValue, DispExpr, EntrySize, EntrySigned, PCRelBaseInstr,
+ FixedEntryLoadInstr);
if (BranchType == IndirectBranchType::UNKNOWN && !MemLocInstr)
return BranchType;
- if (MemLocInstr != &Instruction)
+ if (MemLocInstr && MemLocInstr != &Instruction)
IndexRegNum = BC.MIB->getNoRegister();
if (BC.isAArch64()) {
@@ -911,7 +914,8 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
ArrayStart = static_cast<uint64_t>(DispValue);
}
- if (BaseRegNum == BC.MRI->getProgramCounter())
+ if (BaseRegNum != BC.MIB->getNoRegister() &&
+ BaseRegNum == BC.MRI->getProgramCounter())
ArrayStart += getAddress() + Offset + Size;
if (FixedEntryLoadInstr) {
@@ -992,6 +996,16 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
// Check if there's already a jump table registered at this address.
MemoryContentsType MemType;
if (JumpTable *JT = BC.getJumpTableContainingAddress(ArrayStart)) {
+ if (BC.isRISCV()) {
+ const bool IsRelated = llvm::all_of(JT->Parents, [&](BinaryFunction *BF) {
+ return BC.areRelatedFragments(this, BF);
+ });
+ if (!IsRelated)
+ return IndirectBranchType::UNKNOWN;
+ }
+
+ EntrySize = JT->EntrySize;
+ EntrySigned = JT->EntriesAreSigned;
switch (JT->Type) {
case JumpTable::JTT_NORMAL:
MemType = MemoryContentsType::POSSIBLE_JUMP_TABLE;
@@ -1000,6 +1014,18 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
MemType = MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;
break;
}
+ } else if (EntrySize) {
+ const JumpTable::JumpTableType ExpectedType =
+ BranchType == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE
+ ? JumpTable::JTT_PIC
+ : JumpTable::JTT_NORMAL;
+ const bool IsJumpTable =
+ BC.analyzeJumpTable(ArrayStart, ExpectedType, *this, 0, nullptr,
+ nullptr, EntrySize, EntrySigned);
+ MemType = IsJumpTable ? ExpectedType == JumpTable::JTT_PIC
+ ? MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE
+ : MemoryContentsType::POSSIBLE_JUMP_TABLE
+ : MemoryContentsType::UNKNOWN;
} else {
MemType = BC.analyzeMemoryAt(ArrayStart, *this);
}
@@ -1022,8 +1048,10 @@ BinaryFunction::processIndirectBranch(MCInst &Instruction, unsigned Size,
}
// Convert the instruction into jump table branch.
- const MCSymbol *JTLabel = BC.getOrCreateJumpTable(*this, ArrayStart, JTType);
- BC.MIB->replaceMemOperandDisp(*MemLocInstr, JTLabel, BC.Ctx.get());
+ const MCSymbol *JTLabel = BC.getOrCreateJumpTable(*this, ArrayStart, JTType,
+ EntrySize, EntrySigned);
+ if (MemLocInstr)
+ BC.MIB->replaceMemOperandDisp(*MemLocInstr, JTLabel, BC.Ctx.get());
BC.MIB->setJumpTable(Instruction, ArrayStart, IndexRegNum);
JTSites.emplace_back(Offset, ArrayStart);
@@ -2165,12 +2193,14 @@ bool BinaryFunction::postProcessIndirectBranches(
unsigned BaseRegNum, IndexRegNum;
int64_t DispValue;
const MCExpr *DispExpr;
+ uint64_t EntrySize;
+ bool EntrySigned;
MCInst *PCRelBaseInstr;
MCInst *FixedEntryLoadInstr;
IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
Instr, BB.begin(), II, PtrSize, MemLocInstr, BaseRegNum,
- IndexRegNum, DispValue, DispExpr, PCRelBaseInstr,
- FixedEntryLoadInstr);
+ IndexRegNum, DispValue, DispExpr, EntrySize, EntrySigned,
+ PCRelBaseInstr, FixedEntryLoadInstr);
if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr)
continue;
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 6f588d2b95fd6..cd83b847ad25d 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -13,6 +13,7 @@
#include "bolt/Core/JumpTable.h"
#include "bolt/Core/BinaryFunction.h"
#include "bolt/Core/BinarySection.h"
+#include "bolt/Core/Relocation.h"
#include "llvm/Support/CommandLine.h"
#define DEBUG_TYPE "bolt"
@@ -28,10 +29,11 @@ extern cl::opt<unsigned> Verbosity;
} // namespace opts
bolt::JumpTable::JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize,
- JumpTableType Type, LabelMapType &&Labels,
- BinarySection &Section)
+ bool EntriesAreSigned, JumpTableType Type,
+ LabelMapType &&Labels, BinarySection &Section)
: BinaryData(Symbol, Address, 0, EntrySize, Section), EntrySize(EntrySize),
- OutputEntrySize(EntrySize), Type(Type), Labels(Labels) {}
+ OutputEntrySize(EntrySize), Type(Type),
+ EntriesAreSigned(EntriesAreSigned), Labels(Labels) {}
std::pair<size_t, size_t>
bolt::JumpTable::getEntriesForAddress(const uint64_t Addr) const {
@@ -84,8 +86,11 @@ void bolt::JumpTable::updateOriginal() {
const uint64_t BaseOffset = getAddress() - getSection().getAddress();
uint64_t EntryOffset = BaseOffset;
for (MCSymbol *Entry : Entries) {
- const uint32_t RelType =
- Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+ assert((Type == JTT_PIC || EntrySize == 4 || EntrySize == 8) &&
+ "unsupported absolute jump-table entry size");
+ const uint32_t RelType = Type == JTT_PIC ? Relocation::getPC32()
+ : EntrySize == 4 ? Relocation::getAbs32()
+ : Relocation::getAbs64();
const uint64_t RelAddend =
Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
// Replace existing relocation with the new one to allow any modifications
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index b8354ac91278a..6ad23c3c53a82 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -1012,6 +1012,20 @@ uint32_t Relocation::getPC64() {
}
}
+uint32_t Relocation::getAbs32() {
+ switch (Arch) {
+ default:
+ llvm_unreachable("Unsupported architecture");
+ case Triple::aarch64:
+ return ELF::R_AARCH64_ABS32;
+ case Triple::riscv64:
+ case Triple::riscv32:
+ return ELF::R_RISCV_32;
+ case Triple::x86_64:
+ return ELF::R_X86_64_32;
+ }
+}
+
uint32_t Relocation::getType(const object::RelocationRef &Rel) {
uint64_t RelType = Rel.getType();
assert(isUInt<32>(RelType) && "BOLT relocation types are 32 bits");
diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp
index 39ae4cda145c4..0e891733182ce 100644
--- a/bolt/lib/Passes/IndirectCallPromotion.cpp
+++ b/bolt/lib/Passes/IndirectCallPromotion.cpp
@@ -387,11 +387,13 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
unsigned BaseReg, IndexReg;
int64_t DispValue;
const MCExpr *DispExpr;
+ uint64_t EntrySize;
+ bool EntrySigned;
MutableArrayRef<MCInst> Insts(&BB.front(), &CallInst);
const IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
CallInst, Insts.begin(), Insts.end(), BC.AsmInfo->getCodePointerSize(),
- MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, PCRelBaseOut,
- FixedEntryLoadInstr);
+ MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, EntrySize,
+ EntrySigned, PCRelBaseOut, FixedEntryLoadInstr);
assert(MemLocInstr && "There should always be a load for jump tables");
if (!MemLocInstr)
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 5629b558a2da3..64a0701055440 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -1784,18 +1784,19 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
return Uses;
}
- IndirectBranchType
- analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
- InstructionIterator End, const unsigned PtrSize,
- MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
- unsigned &IndexRegNumOut, int64_t &DispValueOut,
- const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
- MCInst *&FixedEntryLoadInstr) const override {
+ IndirectBranchType analyzeIndirectBranch(
+ MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
+ const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
+ unsigned &IndexRegNumOut, int64_t &DispValueOut,
+ const MCExpr *&DispExprOut, uint64_t &EntrySizeOut, bool &EntrySignedOut,
+ MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInstr) const override {
MemLocInstrOut = nullptr;
BaseRegNumOut = AArch64::NoRegister;
IndexRegNumOut = AArch64::NoRegister;
DispValueOut = 0;
DispExprOut = nullptr;
+ EntrySizeOut = 0;
+ EntrySignedOut = false;
FixedEntryLoadInstr = nullptr;
// An instruction referencing memory used by jump instruction (directly or
@@ -1817,6 +1818,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
MemLocInstrOut = MemLocInstr;
DispValueOut = DispValue;
DispExprOut = DispExpr;
+ EntrySizeOut = ScaleValue;
+ EntrySignedOut = true;
PCRelBaseOut = PCRelBase;
return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;
}
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 2306b462bc4a0..f714e064e3f25 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -257,12 +257,15 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
const unsigned PtrSize, MCInst *&MemLocInstr, unsigned &BaseRegNum,
unsigned &IndexRegNum, int64_t &DispValue, const MCExpr *&DispExpr,
- MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInst) const override {
+ uint64_t &EntrySize, bool &EntrySigned, MCInst *&PCRelBaseOut,
+ MCInst *&FixedEntryLoadInst) const override {
MemLocInstr = nullptr;
BaseRegNum = 0;
IndexRegNum = 0;
DispValue = 0;
DispExpr = nullptr;
+ EntrySize = 0;
+ EntrySigned = false;
PCRelBaseOut = nullptr;
FixedEntryLoadInst = nullptr;
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index 684bedacde3e9..6a6cb8d269bd5 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -2008,13 +2008,12 @@ class X86MCPlusBuilder : public MCPlusBuilder {
SecondInstr, nullptr);
}
- IndirectBranchType
- analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
- InstructionIterator End, const unsigned PtrSize,
- MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
- unsigned &IndexRegNumOut, int64_t &DispValueOut,
- const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
- MCInst *&FixedEntryLoadInst) const override {
+ IndirectBranchType analyzeIndirectBranch(
+ MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
+ const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
+ unsigned &IndexRegNumOut, int64_t &DispValueOut,
+ const MCExpr *&DispExprOut, uint64_t &EntrySizeOut, bool &EntrySignedOut,
+ MCInst *&PCRelBaseOut, MCInst *&FixedEntryLoadInst) const override {
// Try to find a (base) memory location from where the address for
// the indirect branch is loaded. For X86-64 the memory will be specified
// in the following format:
@@ -2041,6 +2040,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
IndexRegNumOut = X86::NoRegister;
DispValueOut = 0;
DispExprOut = nullptr;
+ EntrySizeOut = 0;
+ EntrySignedOut = false;
FixedEntryLoadInst = nullptr;
std::reverse_iterator<InstructionIterator> II(End);
>From 88eb942fc7ba114d5049567c30b749d80df9e719 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Mon, 3 Aug 2026 15:59:04 +0800
Subject: [PATCH 08/10] [BOLT][RISCV] Recognize jump-table dispatch sequences
Follow local register use-def chains to recognize absolute and PIC RISC-V jump-table dispatches, including non-adjacent setup, load, and branch instructions.
Retarget dispatch expressions when a shared jump table is duplicated so every rewritten function references its own table label.
---
bolt/include/bolt/Core/MCPlusBuilder.h | 9 +
bolt/lib/Core/BinaryContext.cpp | 8 +-
bolt/lib/Core/BinaryFunction.cpp | 32 ++
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 399 ++++++++++++++++++-
bolt/test/RISCV/jump-table-shared-anchor.s | 92 +++++
bolt/test/RISCV/jump-table-use-def.s | 120 ++++++
6 files changed, 642 insertions(+), 18 deletions(-)
create mode 100644 bolt/test/RISCV/jump-table-shared-anchor.s
create mode 100644 bolt/test/RISCV/jump-table-use-def.s
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 1e0bdcd54117c..084005783f0bc 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1155,6 +1155,15 @@ class MCPlusBuilder {
return nullptr;
}
+ /// Retarget the reference used by the jump-table dispatch at the end of
+ /// \p InstrWindow from \p OldTarget to \p NewTarget. Targets that need
+ /// architecture-specific multi-instruction matching can override this hook.
+ virtual bool replaceJumpTableReference(
+ MutableArrayRef<MCInst> InstrWindow, const MCSymbol *OldTarget,
+ const MCSymbol *NewTarget, MCContext *Ctx) const {
+ return false;
+ }
+
/// \brief Given a branch instruction try to get the address the branch
/// targets. Return true on success, and the address in Target.
virtual bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 920e0c6c796d9..318863d592137 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -17,6 +17,7 @@
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
@@ -706,13 +707,16 @@ bool BinaryContext::analyzeJumpTable(
<< " -> ");
// Check if there's a proper relocation against the jump table entry.
if (HasRelocations) {
+ const Relocation *Rel = getRelocationAt(EntryAddress);
+ const bool HasRISCVLabelDifference =
+ isRISCV() && Rel && Rel->Type == ELF::R_RISCV_ADD32;
if (Type == JumpTable::JTT_PIC &&
- !DataPCRelocations.count(EntryAddress)) {
+ !DataPCRelocations.count(EntryAddress) && !HasRISCVLabelDifference) {
LLVM_DEBUG(
dbgs() << "FAIL: JTT_PIC table, no relocation for this address\n");
break;
}
- if (Type == JumpTable::JTT_NORMAL && !getRelocationAt(EntryAddress)) {
+ if (Type == JumpTable::JTT_NORMAL && !Rel) {
LLVM_DEBUG(
dbgs()
<< "FAIL: JTT_NORMAL table, no relocation for this address\n");
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 43a128dc17850..3188086b1fa3e 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -4215,6 +4215,38 @@ void BinaryFunction::disambiguateJumpTables(
continue;
if (JumpTables.insert(JT).second)
continue;
+
+ if (BC.isRISCV()) {
+ const uint64_t JTAddress = BC.MIB->getJumpTable(Inst);
+ const uint64_t JTOffset = JTAddress - JT->getAddress();
+ const auto LabelIt = JT->Labels.find(JTOffset);
+ if (LabelIt == JT->Labels.end()) {
+ BC.errs() << "BOLT-ERROR: failed to find RISC-V jump table label at "
+ << "offset 0x" << Twine::utohexstr(JTOffset)
+ << " in function " << *this << '\n';
+ exit(1);
+ }
+
+ const MCSymbol *OldJTLabel = LabelIt->second;
+ uint64_t NewJumpTableID = 0;
+ const MCSymbol *NewJTLabel;
+ std::tie(NewJumpTableID, NewJTLabel) =
+ BC.duplicateJumpTable(*this, JT, OldJTLabel);
+
+ MutableArrayRef<MCInst> InstrWindow(&*BB->begin(), &Inst + 1);
+ if (!BC.MIB->replaceJumpTableReference(
+ InstrWindow, OldJTLabel, NewJTLabel, BC.Ctx.get())) {
+ BC.errs() << "BOLT-ERROR: failed to retarget duplicated RISC-V jump "
+ "table in function "
+ << *this << '\n';
+ exit(1);
+ }
+
+ const uint16_t IndexReg = BC.MIB->getJumpTableIndexReg(Inst);
+ BC.MIB->setJumpTable(Inst, NewJumpTableID, IndexReg, AllocId);
+ continue;
+ }
+
// This instruction is an indirect jump using a jump table, but it is
// using the same jump table of another jump. Try all our tricks to
// extract the jump table symbol and make it point to a new, duplicated JT
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index f714e064e3f25..81c44fac2ab0f 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -10,11 +10,14 @@
//
//===----------------------------------------------------------------------===//
-#include "MCTargetDesc/RISCVMCAsmInfo.h"
#include "MCTargetDesc/RISCVFixupKinds.h"
+#include "MCTargetDesc/RISCVMCAsmInfo.h"
#include "MCTargetDesc/RISCVMCTargetDesc.h"
#include "RISCVMCSymbolizer.h"
#include "bolt/Core/MCPlusBuilder.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInst.h"
@@ -30,6 +33,20 @@ using namespace bolt;
namespace {
class RISCVMCPlusBuilder : public MCPlusBuilder {
+ using LocalUDChain = DenseMap<const MCInst *, SmallVector<MCInst *, 4>>;
+
+ struct JumpTableLoad {
+ const MCInst *Inst{nullptr};
+ uint64_t EntrySize{0};
+ bool EntrySigned{false};
+ int64_t Offset{0};
+ };
+
+ struct ScaledAddress {
+ const MCInst *BaseDef{nullptr};
+ MCPhysReg IndexReg{MCRegister::NoRegister};
+ };
+
bool isRV64() const { return STI->hasFeature(RISCV::Feature64Bit); }
bool isRVE() const { return STI->hasFeature(RISCV::FeatureStdExtE); }
unsigned regSize() const { return isRV64() ? 8 : 4; }
@@ -39,6 +56,255 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
return isRV64() ? RISCV::AMOADD_D : RISCV::AMOADD_W;
}
+ LocalUDChain computeLocalUDChain(const MCInst *CurInstr,
+ InstructionIterator Begin,
+ InstructionIterator End) const {
+ DenseMap<int, MCInst *> RegAliasTable;
+ LocalUDChain Uses;
+
+ auto addInstrOperands = [&](const MCInst &Instr) {
+ for (const MCOperand &Operand : MCPlus::primeOperands(Instr)) {
+ if (!Operand.isReg())
+ continue;
+ Uses[&Instr].push_back(RegAliasTable[Operand.getReg()]);
+ }
+ };
+
+ bool TerminatorSeen = false;
+ for (auto II = Begin; II != End; ++II) {
+ MCInst &Instr = *II;
+ if (isPseudo(Instr) || isNoop(Instr))
+ continue;
+ if (TerminatorSeen) {
+ RegAliasTable.clear();
+ Uses.clear();
+ }
+
+ addInstrOperands(Instr);
+
+ BitVector Regs(RegInfo->getNumRegs(), false);
+ getWrittenRegs(Instr, Regs);
+ for (int Idx : Regs.set_bits())
+ RegAliasTable[Idx] = &Instr;
+
+ TerminatorSeen = isTerminator(Instr);
+ }
+
+ if (CurInstr)
+ addInstrOperands(*CurInstr);
+
+ return Uses;
+ }
+
+ const MCInst *getOperandDef(const MCInst &Inst, unsigned OperandIndex,
+ const LocalUDChain &UDChain) const {
+ if (OperandIndex >= MCPlus::getNumPrimeOperands(Inst) ||
+ !Inst.getOperand(OperandIndex).isReg())
+ return nullptr;
+
+ const auto UsesIt = UDChain.find(&Inst);
+ if (UsesIt == UDChain.end())
+ return nullptr;
+
+ unsigned RegOperandIndex = 0;
+ for (unsigned Index = 0; Index < OperandIndex; ++Index)
+ RegOperandIndex += Inst.getOperand(Index).isReg();
+
+ if (RegOperandIndex >= UsesIt->second.size())
+ return nullptr;
+ return UsesIt->second[RegOperandIndex];
+ }
+
+ const MCInst *followCopies(const MCInst *Def,
+ const LocalUDChain &UDChain) const {
+ SmallPtrSet<const MCInst *, 4> Visited;
+ while (Def && Visited.insert(Def).second) {
+ unsigned SourceOperand = 0;
+ switch (Def->getOpcode()) {
+ default:
+ return Def;
+ case RISCV::ADDI:
+ case RISCV::ORI:
+ if (!Def->getOperand(2).isImm() || Def->getOperand(2).getImm() != 0)
+ return Def;
+ SourceOperand = 1;
+ break;
+ case RISCV::ADD:
+ case RISCV::OR:
+ if (Def->getOperand(1).getReg() == RISCV::X0)
+ SourceOperand = 2;
+ else if (Def->getOperand(2).getReg() == RISCV::X0)
+ SourceOperand = 1;
+ else
+ return Def;
+ break;
+ case RISCV::C_MV:
+ SourceOperand = 1;
+ break;
+ }
+ Def = getOperandDef(*Def, SourceOperand, UDChain);
+ }
+ return Def;
+ }
+
+ static const MCExpr *stripSpecifier(const MCExpr *Expr) {
+ while (const auto *Specifier = dyn_cast_or_null<MCSpecifierExpr>(Expr))
+ Expr = Specifier->getSubExpr();
+ return Expr;
+ }
+
+ const MCExpr *matchJumpTableBase(const MCInst *Def,
+ const LocalUDChain &UDChain) const {
+ Def = followCopies(Def, UDChain);
+ if (!Def)
+ return nullptr;
+
+ if (Def->getOpcode() == RISCV::ADDI || Def->getOpcode() == RISCV::C_ADDI) {
+ Def = followCopies(getOperandDef(*Def, 1, UDChain), UDChain);
+ if (!Def)
+ return nullptr;
+ }
+
+ switch (Def->getOpcode()) {
+ default:
+ return nullptr;
+ case RISCV::LUI:
+ case RISCV::AUIPC:
+ case RISCV::C_LUI:
+ break;
+ }
+
+ if (!Def->getOperand(1).isExpr())
+ return nullptr;
+ const MCExpr *Expr = stripSpecifier(Def->getOperand(1).getExpr());
+ return getTargetSymbolInfo(Expr).first ? Expr : nullptr;
+ }
+
+ bool matchJumpTableLoad(const MCInst *Def, const LocalUDChain &UDChain,
+ JumpTableLoad &Load) const {
+ Def = followCopies(Def, UDChain);
+ if (!Def)
+ return false;
+
+ switch (Def->getOpcode()) {
+ default:
+ return false;
+ case RISCV::LW:
+ case RISCV::C_LW:
+ Load.EntrySize = 4;
+ Load.EntrySigned = isRV64();
+ break;
+ case RISCV::LWU:
+ Load.EntrySize = 4;
+ Load.EntrySigned = false;
+ break;
+ case RISCV::LD:
+ case RISCV::C_LD:
+ // GCC uses full-width label-address arrays as a family of sub-tables
+ // relative to one shared anchor. BOLT cannot move those safely until it
+ // can retarget every LUI/AUIPC + ADDI reference to an interior label.
+ return false;
+ }
+
+ if (!Def->getOperand(2).isImm())
+ return false;
+ Load.Inst = Def;
+ Load.Offset = Def->getOperand(2).getImm();
+ // A non-zero displacement can select an embedded table relative to a
+ // larger anchor object. Moving that table requires retargeting the whole
+ // LUI/AUIPC + ADDI pair to a new interior label, which is not represented
+ // by MemLocInstr today. Reject it instead of moving the wrong sub-table.
+ return Load.Offset == 0;
+ }
+
+ static unsigned getSHXADDScale(unsigned Opcode) {
+ switch (Opcode) {
+ default:
+ return 0;
+ case RISCV::SH1ADD:
+ case RISCV::SH1ADD_UW:
+ return 2;
+ case RISCV::SH2ADD:
+ case RISCV::SH2ADD_UW:
+ return 4;
+ case RISCV::SH3ADD:
+ case RISCV::SH3ADD_UW:
+ return 8;
+ }
+ }
+
+ bool matchScaledAddress(const MCInst *Def, uint64_t EntrySize,
+ const LocalUDChain &UDChain,
+ ScaledAddress &Address) const {
+ Def = followCopies(Def, UDChain);
+ if (!Def)
+ return false;
+
+ if (getSHXADDScale(Def->getOpcode()) == EntrySize) {
+ Address.IndexReg = Def->getOperand(1).getReg();
+ Address.BaseDef = followCopies(getOperandDef(*Def, 2, UDChain), UDChain);
+ return Address.BaseDef != nullptr;
+ }
+
+ if (Def->getOpcode() != RISCV::ADD && Def->getOpcode() != RISCV::C_ADD)
+ return false;
+
+ for (unsigned ShiftOperand : {1U, 2U}) {
+ const unsigned BaseOperand = ShiftOperand == 1 ? 2 : 1;
+ const MCInst *Shift =
+ followCopies(getOperandDef(*Def, ShiftOperand, UDChain), UDChain);
+ if (!Shift)
+ continue;
+ if (Shift->getOpcode() != RISCV::SLLI &&
+ Shift->getOpcode() != RISCV::SLLI_UW &&
+ Shift->getOpcode() != RISCV::C_SLLI)
+ continue;
+ if (!Shift->getOperand(2).isImm() ||
+ (1ULL << Shift->getOperand(2).getImm()) != EntrySize)
+ continue;
+
+ Address.IndexReg = Shift->getOperand(1).getReg();
+ Address.BaseDef =
+ followCopies(getOperandDef(*Def, BaseOperand, UDChain), UDChain);
+ if (Address.BaseDef)
+ return true;
+ }
+ return false;
+ }
+
+ bool areSameJumpTable(const MCExpr *LHS, const MCExpr *RHS) const {
+ return getTargetSymbolInfo(LHS) == getTargetSymbolInfo(RHS);
+ }
+
+ bool replaceJumpTableSymbol(MCInst &Inst, const MCSymbol *OldTarget,
+ const MCSymbol *NewTarget,
+ MCContext *Ctx) const {
+ for (unsigned OpIndex = 0;
+ OpIndex < MCPlus::getNumPrimeOperands(Inst); ++OpIndex) {
+ MCOperand &Operand = Inst.getOperand(OpIndex);
+ if (!Operand.isExpr())
+ continue;
+
+ const MCExpr *Expr = Operand.getExpr();
+ const auto *Specifier = dyn_cast<MCSpecifierExpr>(Expr);
+ const MCExpr *SubExpr = Specifier ? Specifier->getSubExpr() : Expr;
+ const auto [Symbol, Addend] = getTargetSymbolInfo(SubExpr);
+ if (Symbol != OldTarget)
+ continue;
+
+ const MCExpr *NewExpr = MCSymbolRefExpr::create(NewTarget, *Ctx);
+ if (Addend)
+ NewExpr = MCBinaryExpr::createAdd(
+ NewExpr, MCConstantExpr::create(Addend, *Ctx), *Ctx);
+ if (Specifier)
+ NewExpr =
+ MCSpecifierExpr::create(NewExpr, Specifier->getSpecifier(), *Ctx);
+ Operand = MCOperand::createExpr(NewExpr);
+ return true;
+ }
+ return false;
+ }
+
public:
using MCPlusBuilder::MCPlusBuilder;
@@ -269,17 +535,117 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
PCRelBaseOut = nullptr;
FixedEntryLoadInst = nullptr;
- // Check for the following long tail call sequence:
- // 1: auipc xi, %pcrel_hi(sym)
- // jalr zero, %pcrel_lo(1b)(xi)
- if (Instruction.getOpcode() == RISCV::JALR && Begin != End) {
- MCInst &PrevInst = *std::prev(End);
- if (isRISCVCall(PrevInst, Instruction) &&
- Instruction.getOperand(0).getReg() == RISCV::X0)
- return IndirectBranchType::POSSIBLE_TAIL_CALL;
+ (void)PtrSize;
+
+ unsigned TargetOperand;
+ switch (Instruction.getOpcode()) {
+ default:
+ return IndirectBranchType::UNKNOWN;
+ case RISCV::JALR:
+ if (Instruction.getOperand(0).getReg() != RISCV::X0)
+ return IndirectBranchType::UNKNOWN;
+ TargetOperand = 1;
+ break;
+ case RISCV::C_JR:
+ TargetOperand = 0;
+ break;
+ }
+
+ LocalUDChain UDChain = computeLocalUDChain(&Instruction, Begin, End);
+ const MCInst *TargetDef =
+ getOperandDef(Instruction, TargetOperand, UDChain);
+
+ // Check for a long tail call. The local use-def chain makes this robust
+ // against unrelated instructions between AUIPC and JALR.
+ if (Instruction.getOpcode() == RISCV::JALR && TargetDef &&
+ isRISCVCall(*TargetDef, Instruction))
+ return IndirectBranchType::POSSIBLE_TAIL_CALL;
+
+ // Jump-table dispatches use an unmodified register as the JALR target.
+ if (Instruction.getOpcode() == RISCV::JALR &&
+ (!Instruction.getOperand(2).isImm() ||
+ Instruction.getOperand(2).getImm() != 0))
+ return IndirectBranchType::UNKNOWN;
+
+ const MCInst *Root = followCopies(TargetDef, UDChain);
+ if (!Root)
+ return IndirectBranchType::UNKNOWN;
+
+ // PIC tables contain signed 32-bit offsets. Match
+ // add target, loaded-offset, table-base
+ // before the absolute-address form, which branches directly to the load.
+ if (Root->getOpcode() == RISCV::ADD || Root->getOpcode() == RISCV::C_ADD) {
+ for (unsigned LoadOperand : {1U, 2U}) {
+ const unsigned BaseOperand = LoadOperand == 1 ? 2 : 1;
+ JumpTableLoad Load;
+ if (!matchJumpTableLoad(getOperandDef(*Root, LoadOperand, UDChain),
+ UDChain, Load) ||
+ Load.EntrySize != 4)
+ continue;
+
+ const MCExpr *TargetBase = matchJumpTableBase(
+ getOperandDef(*Root, BaseOperand, UDChain), UDChain);
+ if (!TargetBase)
+ continue;
+
+ ScaledAddress Address;
+ if (!matchScaledAddress(getOperandDef(*Load.Inst, 1, UDChain),
+ Load.EntrySize, UDChain, Address))
+ continue;
+ const MCExpr *LoadBase = matchJumpTableBase(Address.BaseDef, UDChain);
+ if (!LoadBase || !areSameJumpTable(TargetBase, LoadBase))
+ continue;
+
+ IndexRegNum = Address.IndexReg;
+ DispValue = Load.Offset;
+ DispExpr = LoadBase;
+ EntrySize = Load.EntrySize;
+ EntrySigned = true;
+ return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;
+ }
}
- return IndirectBranchType::UNKNOWN;
+ // Absolute-address tables branch directly to a loaded 32/64-bit entry:
+ // load target, (table-base + index * entry-size)
+ // jr target
+ JumpTableLoad Load;
+ if (!matchJumpTableLoad(Root, UDChain, Load))
+ return IndirectBranchType::UNKNOWN;
+
+ ScaledAddress Address;
+ if (!matchScaledAddress(getOperandDef(*Load.Inst, 1, UDChain),
+ Load.EntrySize, UDChain, Address))
+ return IndirectBranchType::UNKNOWN;
+
+ const MCExpr *LoadBase = matchJumpTableBase(Address.BaseDef, UDChain);
+ if (!LoadBase)
+ return IndirectBranchType::UNKNOWN;
+
+ BaseRegNum = getNoRegister();
+ IndexRegNum = Address.IndexReg;
+ DispValue = Load.Offset;
+ DispExpr = LoadBase;
+ EntrySize = Load.EntrySize;
+ EntrySigned = Load.EntrySigned;
+ return IndirectBranchType::POSSIBLE_JUMP_TABLE;
+ }
+
+ bool replaceJumpTableReference(
+ MutableArrayRef<MCInst> InstrWindow, const MCSymbol *OldTarget,
+ const MCSymbol *NewTarget, MCContext *Ctx) const override {
+ for (MCInst &Inst : llvm::reverse(InstrWindow)) {
+ switch (Inst.getOpcode()) {
+ default:
+ continue;
+ case RISCV::AUIPC:
+ case RISCV::LUI:
+ case RISCV::C_LUI:
+ break;
+ }
+ if (replaceJumpTableSymbol(Inst, OldTarget, NewTarget, Ctx))
+ return true;
+ }
+ return false;
}
bool convertJmpToTailCall(MCInst &Inst) override {
@@ -385,13 +751,13 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
setInstLabel(InstAUIPC, AUIPCLabel);
Code.emplace_back(std::move(InstAUIPC));
- // Load the call target from the GOT slot using LD on RV64 or LW on RV32.
MCInst InstLoad = MCInstBuilder(loadOpc())
.addReg(PLTScratchReg)
.addReg(PLTScratchReg)
.addImm(0);
// Pair the I-type LD/LW immediate with the label on AUIPC. RISC-V
- // R_RISCV_PCREL_LO12_I relocations name the corresponding HI20 location.
+ // R_RISCV_PCREL_LO12_I relocations name the corresponding HI20 location,
+ // not the final GOT-slot symbol.
setOperandToSymbolRef(InstLoad, /*OpNum=*/2, AUIPCLabel,
/*Addend=*/0, Ctx, ELF::R_RISCV_PCREL_LO12_I);
Code.emplace_back(std::move(InstLoad));
@@ -423,6 +789,11 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
if (!isTerminator(*I) || isTailCall(*I) || !isBranch(*I))
break;
+ // An indirect jump has no symbolic TBB operand. It may be a recognized
+ // jump-table dispatch and must not enter the direct unconditional path.
+ if (isIndirectBranch(*I))
+ return false;
+
// Handle unconditional branches.
if (isUnconditionalBranch(*I)) {
// If any code was seen after this unconditional branch, we've seen
@@ -436,10 +807,6 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
continue;
}
- // Handle conditional branches and ignore indirect branches
- if (isIndirectBranch(*I))
- return false;
-
if (CondBranch == nullptr) {
const MCSymbol *TargetBB = getTargetSymbol(*I);
if (TargetBB == nullptr) {
diff --git a/bolt/test/RISCV/jump-table-shared-anchor.s b/bolt/test/RISCV/jump-table-shared-anchor.s
new file mode 100644
index 0000000000000..ca2a9a93e0f3f
--- /dev/null
+++ b/bolt/test/RISCV/jump-table-shared-anchor.s
@@ -0,0 +1,92 @@
+// REQUIRES: system-linux,target=riscv64{{.*}}
+
+// Do not treat RV64 full-width label-address arrays as movable jump tables.
+// GCC can place multiple arrays at offsets from one shared anchor. Moving the
+// array at offset zero changes the anchor while an unrecognized reference to a
+// later array still relies on the original layout.
+
+// RUN: %clang %cflags64 -march=rv64gc -no-pie \
+// RUN: -Wl,--no-relax,--image-base=0x10000,--section-start=.text=0x20000,--section-start=.rodata=0x30000 \
+// RUN: -o %t %s
+// RUN: %t
+// RUN: llvm-bolt %t -o %t.bolt --jump-tables=move --print-cfg \
+// RUN: --print-only=shared_first,shared_second 2>&1 | FileCheck %s
+// RUN: %t.bolt
+
+// CHECK-LABEL: Binary Function "shared_first"
+// CHECK-NOT: JUMPTABLE
+// CHECK: jr a1 # UNKNOWN CONTROL FLOW
+// CHECK-LABEL: Binary Function "shared_second"
+// CHECK-NOT: JUMPTABLE
+// CHECK: jr a1 # UNKNOWN CONTROL FLOW
+
+ .text
+ .globl _start
+ .type _start, @function
+ .p2align 2
+_start:
+ li a0, 1
+ call shared_second
+ li t0, 41
+ bne a0, t0, .Lfail
+
+ li a0, 0
+ li a7, 93
+ ecall
+.Lfail:
+ li a0, 1
+ li a7, 93
+ ecall
+ .size _start, .-_start
+
+ .globl shared_first
+ .type shared_first, @function
+ .p2align 2
+shared_first:
+ lui a4, %hi(SHARED_ANCHOR)
+ addi a4, a4, %lo(SHARED_ANCHOR)
+ slli a1, a0, 3
+ add a1, a1, a4
+ ld a1, 0(a1)
+ jr a1
+.Lfirst0:
+ li a0, 30
+ ret
+.Lfirst1:
+ li a0, 31
+ ret
+ .size shared_first, .-shared_first
+
+ .globl shared_second
+ .type shared_second, @function
+ .p2align 2
+shared_second:
+ lui a4, %hi(SHARED_ANCHOR)
+ addi a4, a4, %lo(SHARED_ANCHOR)
+ slli a1, a0, 3
+ add a1, a1, a4
+ ld a1, 16(a1)
+ jr a1
+.Lsecond0:
+ li a0, 40
+ ret
+.Lsecond1:
+ li a0, 41
+ ret
+ .size shared_second, .-shared_second
+
+ .section .rodata,"a", at progbits
+ .globl SHARED_ANCHOR
+ .type SHARED_ANCHOR, @object
+ .p2align 3
+SHARED_ANCHOR:
+ .dword .Lfirst0
+ .dword .Lfirst1
+ .size SHARED_ANCHOR, .-SHARED_ANCHOR
+
+ .globl SECOND_JT
+ .type SECOND_JT, @object
+SECOND_JT:
+ .dword .Lsecond0
+ .dword .Lsecond1
+ .size SECOND_JT, .-SECOND_JT
diff --git a/bolt/test/RISCV/jump-table-use-def.s b/bolt/test/RISCV/jump-table-use-def.s
new file mode 100644
index 0000000000000..af1abf52530d7
--- /dev/null
+++ b/bolt/test/RISCV/jump-table-use-def.s
@@ -0,0 +1,120 @@
+// REQUIRES: system-linux,target=riscv64{{.*}}
+
+// Verify that RISC-V jump-table recognition follows the local register
+// use-def chain instead of relying on adjacent instructions. Exercise both
+// GCC-style absolute 32-bit entries and PIC-relative 32-bit entries.
+
+// RUN: %clang %cflags64 -march=rv64gc_zba -no-pie \
+// RUN: -Wl,--no-relax,--image-base=0x10000,--section-start=.text=0x20000,--section-start=.rodata=0x30000 \
+// RUN: -o %t %s
+// RUN: %t
+// RUN: llvm-bolt %t -o %t.bolt --jump-tables=move --print-cfg \
+// RUN: --print-jump-tables --print-only=abs_dispatch,pic_dispatch 2>&1 | \
+// RUN: FileCheck %s
+// RUN: %t.bolt
+
+// RUN: %clang %cflags32 -march=rv32imac_zba -no-pie \
+// RUN: -Wl,--no-relax,--image-base=0x10000,--section-start=.text=0x20000,--section-start=.rodata=0x30000 \
+// RUN: -o %t.rv32 %s
+// RUN: llvm-bolt %t.rv32 -o %t.rv32.bolt --jump-tables=move --print-cfg \
+// RUN: --print-jump-tables --print-only=abs_dispatch,pic_dispatch 2>&1 | \
+// RUN: FileCheck %s
+
+// CHECK-LABEL: Binary Function "abs_dispatch"
+// CHECK: jr a1 # JUMPTABLE @0x30000
+// CHECK-LABEL: Binary Function "pic_dispatch"
+// CHECK: jr a1 # JUMPTABLE @0x3000c
+// CHECK: Jump table ABS_JT for function abs_dispatch
+// CHECK: PIC Jump table PIC_JT for function pic_dispatch
+
+ .text
+ .globl _start
+ .type _start, @function
+ .p2align 2
+_start:
+ li a0, 1
+ call abs_dispatch
+ li t0, 11
+ bne a0, t0, .Lfail
+
+ li a0, 2
+ call pic_dispatch
+ li t0, 22
+ bne a0, t0, .Lfail
+
+ li a0, 0
+ li a7, 93
+ ecall
+.Lfail:
+ li a0, 1
+ li a7, 93
+ ecall
+ .size _start, .-_start
+
+ .globl abs_dispatch
+ .type abs_dispatch, @function
+ .p2align 2
+abs_dispatch:
+ lui a4, %hi(ABS_JT)
+ addi a4, a4, %lo(ABS_JT)
+ li t0, 7 // Unrelated instruction in the def chain.
+ sh2add a1, a0, a4
+ li t1, 8 // Unrelated instruction in the def chain.
+ lw a1, 0(a1)
+ li t2, 9 // The load need not be adjacent to JR.
+ jr a1
+.Labs0:
+ li a0, 10
+ ret
+.Labs1:
+ li a0, 11
+ ret
+.Labs2:
+ li a0, 12
+ ret
+ .size abs_dispatch, .-abs_dispatch
+
+ .globl pic_dispatch
+ .type pic_dispatch, @function
+ .p2align 2
+pic_dispatch:
+.Lpcrel_hi:
+ auipc a4, %pcrel_hi(PIC_JT)
+ addi a4, a4, %pcrel_lo(.Lpcrel_hi)
+ li t0, 7 // Unrelated instruction in the def chain.
+ slli a1, a0, 2
+ add a1, a1, a4
+ li t1, 8 // Unrelated instruction in the def chain.
+ lw a1, 0(a1)
+ li t2, 9 // Separate the load, ADD, and JR.
+ add a1, a1, a4
+ jr a1
+.Lpic0:
+ li a0, 20
+ ret
+.Lpic1:
+ li a0, 21
+ ret
+.Lpic2:
+ li a0, 22
+ ret
+ .size pic_dispatch, .-pic_dispatch
+
+ .section .rodata,"a", at progbits
+ .globl ABS_JT
+ .type ABS_JT, @object
+ .p2align 2
+ABS_JT:
+ .word .Labs0
+ .word .Labs1
+ .word .Labs2
+ .size ABS_JT, .-ABS_JT
+
+ .globl PIC_JT
+ .type PIC_JT, @object
+ .p2align 2
+PIC_JT:
+ .word .Lpic0 - PIC_JT
+ .word .Lpic1 - PIC_JT
+ .word .Lpic2 - PIC_JT
+ .size PIC_JT, .-PIC_JT
>From efde9e4720159720b6abf6f1a0e7ce320c5a7d98 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Mon, 3 Aug 2026 15:59:47 +0800
Subject: [PATCH 09/10] [BOLT][RISCV] Rewrite call pairs across basic-block
boundaries
---
bolt/lib/Passes/FixRISCVCallsPass.cpp | 66 +++++++++++++++++++--------
bolt/test/RISCV/relax.s | 4 +-
bolt/test/RISCV/reloc-call-split-bb.s | 42 +++++++++++++++++
3 files changed, 89 insertions(+), 23 deletions(-)
create mode 100644 bolt/test/RISCV/reloc-call-split-bb.s
diff --git a/bolt/lib/Passes/FixRISCVCallsPass.cpp b/bolt/lib/Passes/FixRISCVCallsPass.cpp
index 6b73bd6854c9d..f1d6a749786fd 100644
--- a/bolt/lib/Passes/FixRISCVCallsPass.cpp
+++ b/bolt/lib/Passes/FixRISCVCallsPass.cpp
@@ -9,8 +9,6 @@
#include "bolt/Passes/FixRISCVCallsPass.h"
#include "bolt/Core/ParallelUtilities.h"
-#include <iterator>
-
using namespace llvm;
namespace llvm {
@@ -21,8 +19,17 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) {
auto &MIB = BC.MIB;
auto *Ctx = BC.Ctx.get();
+ MCInst *Previous = nullptr;
+ BinaryBasicBlock *PreviousBB = nullptr;
for (auto &BB : BF) {
for (auto II = BB.begin(); II != BB.end();) {
+ // CFI and other zero-sized pseudo instructions do not break an
+ // AUIPC/JALR pair in the input instruction stream.
+ if (MIB->isPseudo(*II)) {
+ ++II;
+ continue;
+ }
+
if (MIB->isCall(*II) && !MIB->isIndirectCall(*II)) {
auto *Target = MIB->getTargetSymbol(*II);
assert(Target && "Cannot find call target");
@@ -36,35 +43,54 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) {
MIB->createCall(*II, Target, Ctx);
MIB->moveAnnotations(std::move(OldCall), *II);
+ Previous = &*II;
+ PreviousBB = &BB;
++II;
continue;
}
- auto NextII = std::next(II);
-
- if (NextII == BB.end())
- break;
-
- if (MIB->isRISCVCall(*II, *NextII)) {
- auto *Target = MIB->getTargetSymbol(*II);
+ // A label, secondary entry point, or CFG boundary may split an
+ // AUIPC/JALR call pair across two basic blocks. Keep the previous real
+ // instruction across block boundaries so that the pair is still
+ // rewritten atomically. Otherwise the old JALR immediate remains in the
+ // encoding and can be ORed with the new R_RISCV_CALL_PLT fixup.
+ if (Previous && MIB->isRISCVCall(*Previous, *II)) {
+ auto *Target = MIB->getTargetSymbol(*Previous);
assert(Target && "Cannot find call target");
- MCInst OldCall = *NextII;
+ MCInst OldCall = *II;
auto L = BC.scopeLock();
- MIB->createNoop(*II);
-
- if (MIB->isTailCall(*NextII))
- MIB->createTailCall(*NextII, Target, Ctx);
- else
- MIB->createCall(*NextII, Target, Ctx);
-
- MIB->moveAnnotations(std::move(OldCall), *NextII);
-
- II = std::next(NextII);
+ if (PreviousBB == &BB) {
+ // Keep the original JALR offset annotation on the combined call, but
+ // emit the pseudo at the AUIPC position and remove the JALR. This
+ // preserves profile attribution without adding an executed NOP to
+ // every long call.
+ if (MIB->isTailCall(*II))
+ MIB->createTailCall(*Previous, Target, Ctx);
+ else
+ MIB->createCall(*Previous, Target, Ctx);
+ MIB->moveAnnotations(std::move(OldCall), *Previous);
+ II = BB.eraseInstruction(II);
+ } else {
+ // Keep split pairs in their original basic blocks. Moving the call
+ // across a CFG boundary would invalidate block-level control-flow
+ // information.
+ MIB->createNoop(*Previous);
+ if (MIB->isTailCall(*II))
+ MIB->createTailCall(*II, Target, Ctx);
+ else
+ MIB->createCall(*II, Target, Ctx);
+ MIB->moveAnnotations(std::move(OldCall), *II);
+ ++II;
+ }
+ Previous = nullptr;
+ PreviousBB = nullptr;
continue;
}
+ Previous = &*II;
+ PreviousBB = &BB;
++II;
}
}
diff --git a/bolt/test/RISCV/relax.s b/bolt/test/RISCV/relax.s
index 41124751f38e8..74f049b8f8dd9 100644
--- a/bolt/test/RISCV/relax.s
+++ b/bolt/test/RISCV/relax.s
@@ -12,15 +12,13 @@
// CHECK: Binary Function "_start" after fix-riscv-calls {
// CHECK: call near_f
-// CHECK-NEXT: nop
// CHECK-NEXT: call far_f
// CHECK-NEXT: tail near_f
// OBJDUMP: 0000000000600000 <_start>:
// OBJDUMP-NEXT: jal 0x600040 <near_f>
-// OBJDUMP-NEXT: nop
// OBJDUMP-NEXT: auipc ra, 0x200
-// OBJDUMP-NEXT: jalr 0x78(ra)
+// OBJDUMP-NEXT: jalr 0x7c(ra)
// OBJDUMP-NEXT: j 0x600040 <near_f>
// OBJDUMP: 0000000000600040 <near_f>:
// OBJDUMP: 0000000000800080 <far_f>:
diff --git a/bolt/test/RISCV/reloc-call-split-bb.s b/bolt/test/RISCV/reloc-call-split-bb.s
new file mode 100644
index 0000000000000..fffdc78c408c6
--- /dev/null
+++ b/bolt/test/RISCV/reloc-call-split-bb.s
@@ -0,0 +1,42 @@
+// Check that FixRISCVCalls rewrites an AUIPC/JALR pair even when a branch
+// target splits the pair across two basic blocks.
+
+// RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj -o %t.o %s
+// RUN: ld.lld --emit-relocs -o %t %t.o
+// RUN: llvm-bolt --print-cfg --print-fix-riscv-calls --print-only=_start \
+// RUN: -o %t.bolt %t | FileCheck %s
+// RUN: llvm-objdump -d %t.bolt | FileCheck --check-prefix=OBJDUMP %s
+
+ .text
+ .option norvc
+
+ .globl target
+ .p2align 2
+target:
+ ret
+ .size target, .-target
+
+ .globl _start
+ .p2align 2
+_start:
+ // This branch is never taken, but makes .Ljalr a basic-block entry.
+ bne zero, zero, .Ljalr
+.Lcall:
+ auipc ra, 0
+ .reloc .Lcall, R_RISCV_CALL_PLT, target
+.Ljalr:
+ jalr ra, ra, 0
+ ret
+ .size _start, .-_start
+
+// CHECK-LABEL: Binary Function "_start" after building cfg {
+// CHECK: auipc ra, target
+// CHECK: jalr
+
+// CHECK-LABEL: Binary Function "_start" after fix-riscv-calls {
+// CHECK: nop
+// CHECK: call target
+
+// OBJDUMP-LABEL: <_start>:
+// OBJDUMP: nop
+// OBJDUMP-NEXT: jal
>From 897b6d69c56839ca05c2172a11f67e07ce5d4091 Mon Sep 17 00:00:00 2001
From: Thrrreeeee <1379998393 at qq.com>
Date: Mon, 3 Aug 2026 16:00:45 +0800
Subject: [PATCH 10/10] [BOLT][RISCV][NFC] Document atomic-add operand order
Clarify that RISC-V AMO operands are emitted as destination, value, and address registers.
---
bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index 81c44fac2ab0f..c65d379b24499 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -1059,6 +1059,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
void atomicAdd(MCInst &Inst, MCPhysReg RegAtomic, MCPhysReg RegTo,
MCPhysReg RegCnt) const {
+ // AMO operands are ordered as rd, rs2 (value), rs1 (address).
Inst = MCInstBuilder(atomicAddOpc())
.addReg(RegAtomic)
.addReg(RegCnt)
More information about the llvm-commits
mailing list