[llvm-branch-commits] [lld] ELF: CFI jump table relaxation. (PR #147424)
Fangrui Song via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat May 23 21:39:43 PDT 2026
================
@@ -312,6 +313,196 @@ bool X86_64::deleteFallThruJmpInsn(InputSection &is,
return true;
}
+void X86_64::relaxCFIJumpTables() const {
+ // Relax CFI jump tables.
+ // - Split jump table into pieces and place target functions inside the jump
+ // table if small enough.
+ // - Move jump table before last called function and delete last branch
+ // instruction.
+ DenseMap<InputSection *, SmallVector<InputSection *, 0>> sectionReplacements;
+ SmallVector<InputSection *, 0> storage;
+ for (OutputSection *osec : ctx.outputSections) {
+ if (!(osec->flags & SHF_EXECINSTR))
+ continue;
+ for (InputSection *sec : getInputSections(*osec, storage)) {
+ if (sec->type != SHT_LLVM_CFI_JUMP_TABLE || sec->entsize == 0 ||
+ sec->size % sec->entsize != 0)
+ continue;
+
+ // We're going to replace the jump table with this list of sections. This
+ // list will be made up of slices of the original section and function
+ // bodies that were moved into the jump table.
+ SmallVector<InputSection *, 0> replacements;
+
+ // r is the only relocation in a jump table entry. Figure out whether it
+ // is a branch pointing to the start of a statically known section that
+ // hasn't already been moved while processing a different jump table
+ // section, and if so return it.
+ auto getMovableSection = [&](Relocation &r) -> InputSection * {
+ if (r.type != R_X86_64_PC32 && r.type != R_X86_64_PLT32)
+ return nullptr;
+ auto *sym = dyn_cast<Defined>(r.sym);
+ if (!sym || sym->isPreemptible || sym->isGnuIFunc() ||
+ sym->value + r.addend != -4ull)
+ return nullptr;
+ auto *target = dyn_cast_or_null<InputSection>(sym->section);
+ if (!target || sectionReplacements.count(target))
+ return nullptr;
+ return target;
+ };
+
+ // Figure out the movable section for the last entry. We do this first
+ // because the last entry controls which output section the jump table is
+ // placed into, which affects move eligibility for other sections.
+ auto *lastSec = [&]() -> InputSection * {
+ // If the jump table section is more aligned than the entry size, skip
+ // this because there's no guarantee that we'll be able to emit a
+ // padding section that places the last entry at a correctly aligned
+ // address.
+ if (sec->addralign > sec->entsize)
+ return nullptr;
+
+ auto rels = sec->relocs();
+ if (rels.empty() || rels.back().offset < sec->size - sec->entsize)
+ return nullptr;
+ if (rels.size() >= 2 &&
+ rels[rels.size() - 2].offset >= sec->size - sec->entsize)
+ return nullptr;
+ return getMovableSection(rels.back());
+ }();
+ OutputSection *targetOutputSec;
+ if (lastSec) {
+ // If the last section is more aligned than the jump table, we need
+ // to emit a padding section before the jump table to ensure that the
+ // last section ends up at the correct alignment.
+ if (lastSec->addralign > sec->addralign) {
+ // We need to add enough padding to make this equal to zero.
+ size_t mod = (sec->size - sec->entsize) % lastSec->addralign;
+ if (mod != 0) {
+ auto *pad = make<PaddingSection>(ctx, lastSec->addralign - mod,
+ lastSec->getParent());
+ pad->addralign = lastSec->addralign;
+ replacements.push_back(pad);
+ } else {
+ sec->addralign = lastSec->addralign;
+ }
+ }
+
+ // We've already decided to move the output section so make sure that we
+ // don't try to move it again.
+ sectionReplacements[lastSec] = {};
+ targetOutputSec = lastSec->getParent();
+ } else {
+ targetOutputSec = sec->getParent();
+ }
+
+ // First, push the original jump table section. This is only so that it
+ // can act as a relocation target. Later on, we will set the size of the
+ // jump table section to 0 so that the slices and moved function bodies
+ // become the actual relocation targets.
+ replacements.push_back(sec);
+
+ // Add the slice [begin, end) of the original section to the replacement
+ // list. [rbegin, rend) is the slice of the relocation list that covers
+ // [begin, end).
+ auto addSectionSlice = [&](size_t begin, size_t end, Relocation *rbegin,
+ Relocation *rend) {
+ auto *slice = make<InputSection>(
+ sec->file, sec->name, sec->type, sec->flags, sec->entsize,
+ sec->entsize,
+ sec->contentMaybeDecompress().slice(begin, end - begin));
+ for (const Relocation &r : ArrayRef<Relocation>(rbegin, rend)) {
+ slice->relocations.push_back(
+ Relocation{r.expr, r.type, r.offset - begin, r.addend, r.sym});
+ }
+ replacements.push_back(slice);
+ };
+
+ // Walk the jump table entries other than the last one looking for
+ // sections that are small enough to be moved into the jump table and in
+ // the same section as the jump table's destination.
+ size_t begin = 0, cur = 0;
+ Relocation *rbegin = sec->relocs().begin(), *rcur = rbegin;
+ while (cur != sec->size - sec->entsize) {
+ size_t next = cur + sec->entsize;
+ Relocation *rnext = rcur;
+ while (rnext != sec->relocs().end() && rnext->offset < next)
+ ++rnext;
+ if (rcur + 1 == rnext) {
+ if (InputSection *target = getMovableSection(*rcur);
+ target && target->size != 0 && target->size <= sec->entsize &&
+ target->addralign <= sec->entsize &&
+ target->getParent() == targetOutputSec) {
+ // Okay, we found a small enough section. Move it into the jump
+ // table. First add a slice for the unmodified jump table entries
+ // before this one. This slice may be of zero size if two
+ // consecutive functions are moved to the jump table, and is
+ // used to correctly align the target function.
+ addSectionSlice(begin, cur, rbegin, rcur);
+ // Add the target to our replacement list, and set the target's
+ // replacement list to the empty list. This removes it from its
+ // original position and adds it here, as well as causing
+ // future getMovableSection() queries to return nullptr.
+ replacements.push_back(target);
+ sectionReplacements[target] = {};
+ begin = next;
+ rbegin = rnext;
+ }
+ }
+ cur = next;
+ rcur = rnext;
+ }
+
+ // Finally, process the last entry. If it is movable, move the entire
+ // jump table behind it and delete the last entry (so that the last
+ // function's body acts as the last jump table entry), otherwise leave the
+ // jump table where it is and keep the last entry.
+ if (lastSec) {
+ addSectionSlice(begin, cur, rbegin, rcur);
+ replacements.push_back(lastSec);
+ sectionReplacements[sec] = {};
+ for (auto *s : replacements)
+ s->parent = lastSec->parent;
+ sectionReplacements[lastSec] = std::move(replacements);
+ } else {
+ addSectionSlice(begin, sec->size, rbegin, sec->relocs().end());
+ for (auto *s : replacements)
+ s->parent = sec->parent;
+ sectionReplacements[sec] = std::move(replacements);
+ }
+
+ // Everything from the original section has been recreated, so delete the
+ // original contents.
+ sec->relocations.clear();
+ sec->size = 0;
+ }
+ }
+
+ if (sectionReplacements.empty())
+ return;
+
+ // Now that we have the complete mapping of replacements, go through the input
+ // section lists and apply the replacements.
+ for (OutputSection *osec : ctx.outputSections) {
+ if (!(osec->flags & SHF_EXECINSTR))
+ continue;
+ for (SectionCommand *cmd : osec->commands) {
+ auto *isd = dyn_cast<InputSectionDescription>(cmd);
+ if (!isd)
+ continue;
+ SmallVector<InputSection *> newSections;
+ for (auto *sec : isd->sections) {
+ auto i = sectionReplacements.find(sec);
+ if (i == sectionReplacements.end())
+ newSections.push_back(sec);
+ else
+ newSections.append(i->second.begin(), i->second.end());
+ }
+ isd->sections = std::move(newSections);
+ }
+ }
+}
+
----------------
MaskRay wrote:
L577 at main, `stable_sort(sec.relocs(), ...` is gated on `--branch-to-branch`, while this patch introduces another options.
Is it useful to have another `--relax-*` option?
https://github.com/llvm/llvm-project/pull/147424
More information about the llvm-branch-commits
mailing list