[llvm] [bolt] Simplify rodata/literal load for X86_64 & AArch64 (PR #179474)
Alexey Moksyakov via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 22:46:56 PDT 2026
https://github.com/yavtuk updated https://github.com/llvm/llvm-project/pull/179474
>From 27089b849a52d75b3a6fe92c5bf401882e882951 Mon Sep 17 00:00:00 2001
From: yavtuk <yavtuk at ya.ru>
Date: Fri, 11 Sep 2026 08:25:15 +0300
Subject: [PATCH] [bolt] Simplify rodata/literal load for X86_64 & AArch64
This patch enables simplify rodata loads by default for X86_64 and AArch64.
For AArch64, this is the fix related to ldr literal instruction where constant
island is moved to the end of a function and the target address can be out
of available range +/- 1MB. For AArch64, the available data sizes are 4, 8 bytes,
for X86_64 available sizes 2, 4 and 8 bytes.
Note: SimplifyRODataLoads is disabled for non-relocation mode.
---
bolt/include/bolt/Core/MCPlusBuilder.h | 9 ++
bolt/lib/Passes/BinaryPasses.cpp | 53 ++++++++----
bolt/lib/Rewrite/BinaryPassManager.cpp | 12 +--
bolt/lib/Rewrite/RewriteInstance.cpp | 10 +++
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 27 ++++++
bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 14 ++++
bolt/test/AArch64/materialize-constant.s | 82 +++++++++++++++++++
7 files changed, 186 insertions(+), 21 deletions(-)
create mode 100644 bolt/test/AArch64/materialize-constant.s
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index e55db33834ee8..79298c57985f2 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1958,6 +1958,15 @@ class MCPlusBuilder {
return {};
}
+ /// Materializing \p ConstantData value in the target register of \p Inst
+ virtual InstructionListType materializeConstant(BinaryContext &BC,
+ const MCInst &Inst,
+ StringRef ConstantData,
+ uint64_t Offset) const {
+ llvm_unreachable("not implemented");
+ return {};
+ }
+
/// Creates a new unconditional branch instruction in Inst and set its operand
/// to TBB.
virtual void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index f2b664181f2ba..4bb3f0988de6b 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -1262,7 +1262,8 @@ bool SimplifyRODataLoads::simplifyRODataLoads(BinaryFunction &BF) {
uint64_t NumDynamicLocalLoadsFound = 0;
for (BinaryBasicBlock *BB : BF.getLayout().blocks()) {
- for (MCInst &Inst : *BB) {
+ for (auto It = BB->begin(); It != BB->end(); ++It) {
+ MCInst &Inst = *It;
unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = BC.MII->get(Opcode);
@@ -1301,28 +1302,49 @@ bool SimplifyRODataLoads::simplifyRODataLoads(BinaryFunction &BF) {
}
// Get the contents of the section containing the target address of the
- // memory operand. We are only interested in read-only sections.
+ // memory operand. We are only interested in read-only sections for X86,
+ // for aarch64 the sections can be read-only or executable.
ErrorOr<BinarySection &> DataSection =
BC.getSectionForAddress(TargetAddress);
if (!DataSection || DataSection->isWritable())
continue;
+ if (DataSection->isText()) {
+ // If data is not part of a function, check if it is part of a global CI
+ // Do not proceed if there aren't data markers for CIs
+ BinaryFunction *TargetBF =
+ BC.getBinaryFunctionContainingAddress(TargetAddress,
+ /*CheckPastEnd*/ false,
+ /*UseMaxSize*/ true);
+ const bool IsInsideFunc =
+ TargetBF && TargetBF->isInConstantIsland(TargetAddress);
+
+ auto CIEndIter = BC.AddressToConstantIslandMap.end();
+ auto CIIter = BC.AddressToConstantIslandMap.find(TargetAddress);
+ if (!IsInsideFunc && CIIter == CIEndIter)
+ continue;
+ }
+
if (BC.getRelocationAt(TargetAddress) ||
BC.getDynamicRelocationAt(TargetAddress))
continue;
- uint32_t Offset = TargetAddress - DataSection->getAddress();
- StringRef ConstantData = DataSection->getContents();
-
++NumLocalLoadsFound;
if (BB->hasProfile())
NumDynamicLocalLoadsFound += BB->getExecutionCount();
- if (MIB->replaceMemOperandWithImm(Inst, ConstantData, Offset)) {
- ++NumLocalLoadsSimplified;
- if (BB->hasProfile())
- NumDynamicLocalLoadsSimplified += BB->getExecutionCount();
- }
+ uint32_t Offset = TargetAddress - DataSection->getAddress();
+ StringRef ConstantData = DataSection->getContents();
+ const InstructionListType Instrs =
+ MIB->materializeConstant(BC, Inst, ConstantData, Offset);
+ if (Instrs.empty())
+ continue;
+
+ It = std::next(BB->replaceInstruction(It, Instrs), Instrs.size() - 1);
+
+ ++NumLocalLoadsSimplified;
+ if (BB->hasProfile())
+ NumDynamicLocalLoadsSimplified += BB->getExecutionCount();
}
}
@@ -1341,12 +1363,11 @@ Error SimplifyRODataLoads::runOnFunctions(BinaryContext &BC) {
Modified.insert(&Function);
}
- BC.outs() << "BOLT-INFO: simplified " << NumLoadsSimplified << " out of "
- << NumLoadsFound << " loads from a statically computed address.\n"
- << "BOLT-INFO: dynamic loads simplified: "
- << NumDynamicLoadsSimplified << "\n"
- << "BOLT-INFO: dynamic loads found: " << NumDynamicLoadsFound
- << "\n";
+ if (opts::Verbosity > 0 || NumLoadsSimplified)
+ BC.outs() << "BOLT-INFO: simplified " << NumLoadsSimplified << " out of "
+ << NumLoadsFound << " loads from statically computed addresses\n"
+ << "BOLT-INFO: simplified " << NumDynamicLoadsSimplified
+ << " out of " << NumDynamicLoadsFound << " dynamic loads\n";
return Error::success();
}
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index e61297f94ff98..dbbedea2f70bc 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -241,11 +241,11 @@ static cl::opt<bool> SimplifyConditionalTailCalls(
cl::desc("simplify conditional tail calls by removing unnecessary jumps"),
cl::init(true), cl::cat(BoltOptCategory));
-static cl::opt<bool> SimplifyRODataLoads(
+cl::opt<bool> SimplifyRODataLoads(
"simplify-rodata-loads",
cl::desc("simplify loads from read-only sections by replacing the memory "
"operand with the constant found in the corresponding section"),
- cl::cat(BoltOptCategory));
+ cl::init(false), cl::cat(BoltOptCategory));
static cl::list<std::string>
SpecializeMemcpy1("memcpy1-spec",
@@ -442,9 +442,11 @@ Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
std::make_unique<JTFootprintReduction>(PrintJTFootprintReduction),
opts::JTFootprintReductionFlag);
- Manager.registerPass(
- std::make_unique<SimplifyRODataLoads>(PrintSimplifyROLoads),
- opts::SimplifyRODataLoads);
+ if (!BC.isRISCV()) {
+ Manager.registerPass(
+ std::make_unique<SimplifyRODataLoads>(PrintSimplifyROLoads),
+ opts::SimplifyRODataLoads);
+ }
Manager.registerPass(std::make_unique<RegReAssign>(PrintRegReAssign),
opts::RegReAssign);
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 385b9e736fe64..cede4d2c946b5 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -89,6 +89,7 @@ extern cl::list<std::string> PrintOnly;
extern cl::opt<std::string> PrintOnlyFile;
extern cl::list<std::string> ReorderData;
extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
+extern cl::opt<bool> SimplifyRODataLoads;
extern cl::opt<bool> TerminalHLT;
extern cl::opt<bool> TerminalTrap;
extern cl::opt<bool> TimeBuild;
@@ -2748,6 +2749,15 @@ void RewriteInstance::adjustCommandLineOptions() {
if (!opts::TerminalTrap.getNumOccurrences())
opts::TerminalTrap = false;
}
+
+ if (opts::SimplifyRODataLoads &&
+ (BC->isRISCV() || (BC->isAArch64() && !BC->HasRelocations))) {
+ // TODO: For RISCV, the optimization is not implemented yet.
+ // For AArch64, the one is disabled to avoid increasing
+ // the output functions size in non relocs mode.
+ opts::SimplifyRODataLoads = false;
+ BC->outs() << "BOLT-INFO: simplify rodata loads pass is disabled\n";
+ }
}
namespace {
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 253893709a166..9f88f4768a441 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -3665,6 +3665,33 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
return Insts;
}
+ InstructionListType materializeConstant(BinaryContext &BC, const MCInst &Inst,
+ StringRef ConstantData,
+ uint64_t Offset) const override {
+ // Size in bytes that Inst loads from memory.
+ uint8_t DataSize = 0;
+ switch (Inst.getOpcode()) {
+ case AArch64::LDRWl:
+ DataSize = 4;
+ break;
+ case AArch64::LDRXl:
+ DataSize = 8;
+ break;
+ default:
+ return InstructionListType{};
+ }
+
+ if (Offset + DataSize > ConstantData.size())
+ return InstructionListType{};
+
+ DataExtractor DE(ConstantData, BC.AsmInfo->isLittleEndian());
+ const uint64_t Imm = DE.getUnsigned(&Offset, DataSize);
+
+ const MCPhysReg Dest = Inst.getOperand(0).getReg();
+
+ return createLoadImmediate(Dest, Imm);
+ }
+
std::optional<Relocation>
createRelocation(const MCFixup &Fixup,
const MCAsmBackend &MAB) const override {
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index e5930f57b23be..f2a52e188f658 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -1489,6 +1489,20 @@ class X86MCPlusBuilder : public MCPlusBuilder {
return true;
}
+ InstructionListType materializeConstant(BinaryContext &BC, const MCInst &Inst,
+ StringRef ConstantData,
+ uint64_t Offset) const override {
+ InstructionListType Instrs;
+ MCInst InstCopy = Inst;
+
+ if (!replaceMemOperandWithImm(InstCopy, ConstantData, Offset))
+ return Instrs;
+
+ Instrs.push_back(std::move(InstCopy));
+
+ return Instrs;
+ }
+
/// TODO: this implementation currently works for the most common opcodes that
/// load from memory. It can be extended to work with memory store opcodes as
/// well as more memory load opcodes.
diff --git a/bolt/test/AArch64/materialize-constant.s b/bolt/test/AArch64/materialize-constant.s
new file mode 100644
index 0000000000000..a6e0d3060f10e
--- /dev/null
+++ b/bolt/test/AArch64/materialize-constant.s
@@ -0,0 +1,82 @@
+// this test checks a load literal instructions changed to movk
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags %s -Wa,--defsym,CIBIGFUNC=1 -Wl,-q -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --simplify-rodata-loads \
+# RUN: --lite=0 --keep-nops --eliminate-unreachable=false \
+# RUN: | FileCheck %s --check-prefix=CHECK-LOGS
+# RUN: llvm-objdump -d --disassemble-symbols=foo %t.bolt \
+# RUN: | FileCheck %s --check-prefix=CHECK-INSTR
+
+# RUN: %clang %cflags %s -Wa,--defsym,CIOUTSIDEFUNC=1 -Wl,-q -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.bolt --simplify-rodata-loads \
+# RUN: --lite=0 --keep-nops --eliminate-unreachable=false \
+# RUN: | FileCheck %s --check-prefix=CHECK-LOGS
+# RUN: llvm-objdump -d --disassemble-symbols=foo %t.bolt \
+# RUN: | FileCheck %s --check-prefix=CHECK-INSTR
+
+# CHECK-LOGS: simplified 2 out of 2 loads
+
+# CHECK-INSTR: mov w23, #0x64
+# CHECK-INSTR: mov x24, #0xcccc
+# CHECK-INSTR-NEXT: movk x24, #0x3333, lsl #16
+# CHECK-INSTR-NEXT: movk x24, #0x5555, lsl #32
+# CHECK-INSTR-NEXT: movk x24, #0xaaaa, lsl #48
+
+ .text
+ .align 4
+ .local foo
+ .type foo, %function
+foo:
+ stp x29, x30, [sp, #-32]!
+ stp x19, x20, [sp, #16]
+ mov x29, sp
+
+ mov w19, #0 // counter = 0
+ mov w22, #0 // result = 0
+
+ ldr w23, .Llimit
+ ldr x24, .LStep
+
+.ifdef CIBIGFUNC
+ b .LStub
+.LConstants:
+ .Llimit: .word 100
+ .LStep: .xword 0xaaaa55553333cccc
+.LStub:
+// For AArch64, there is the issue related to emitting a constant
+// island to the end of a function, ldr literal instruction can be
+// out of available address range when the function size is ~1MB.
+.rep 0x40000
+ nop
+.endr
+ b .Lreturn_point
+.endif
+
+.Lreturn_point:
+ ldp x19, x20, [sp, #16]
+ ldp x29, x30, [sp], #32
+ ret
+.size foo, .-foo
+
+.ifdef CIOUTSIDEFUNC
+.LConstants:
+ .Llimit: .word 100
+ .LStep: .xword 0xaaaa55553333cccc
+.endif
+
+
+ .global main
+ .type main, %function
+main:
+# Dummy relocation to force relocation mode
+.reloc 0, R_AARCH64_NONE
+ mov x0, #0
+ bl foo
+ mov x0, 0
+ mov w8, #93
+ svc #0
+
+.size main, .-main
+
More information about the llvm-commits
mailing list