[lld] 5d3bd7f - [ELF] Move gotIndex/pltIndex/globalDynIndex to SymbolAux
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sun Jan 9 13:43:31 PST 2022
Author: Fangrui Song
Date: 2022-01-09T13:43:27-08:00
New Revision: 5d3bd7f36092c88265dc698a970078184425f67c
URL: https://github.com/llvm/llvm-project/commit/5d3bd7f36092c88265dc698a970078184425f67c
DIFF: https://github.com/llvm/llvm-project/commit/5d3bd7f36092c88265dc698a970078184425f67c.diff
LOG: [ELF] Move gotIndex/pltIndex/globalDynIndex to SymbolAux
to decrease sizeof(SymbolUnion) by 8 on ELF64 platforms.
Symbols needing such information are typically 1% or fewer (5134 out of 560520
when linking clang, 19898 out of 5550705 when linking chrome). Storing them
elsewhere can decrease memory usage and symbol initialization time.
There is a ~0.8% saving on max RSS when linking a large program.
Future direction:
* Move some of dynsymIndex/verdefIndex/versionId to SymbolAux
* Support mixed TLSDESC and TLS GD without increasing sizeof(SymbolUnion)
Reviewed By: peter.smith
Differential Revision: https://reviews.llvm.org/D116281
Added:
Modified:
lld/ELF/Arch/PPC.cpp
lld/ELF/Arch/PPC64.cpp
lld/ELF/Arch/X86.cpp
lld/ELF/Arch/X86_64.cpp
lld/ELF/Driver.cpp
lld/ELF/Relocations.cpp
lld/ELF/Symbols.cpp
lld/ELF/Symbols.h
lld/ELF/SyntheticSections.cpp
Removed:
################################################################################
diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index e28b623294945..97e4d66331381 100644
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -192,7 +192,7 @@ void PPC::writeGotHeader(uint8_t *buf) const {
void PPC::writeGotPlt(uint8_t *buf, const Symbol &s) const {
// Address of the symbol resolver stub in .glink .
- write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.pltIndex);
+ write32(buf, in.plt->getVA() + in.plt->headerSize + 4 * s.getPltIdx());
}
bool PPC::needsThunk(RelExpr expr, RelType type, const InputFile *file,
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index d5e73ab9ec979..69a9118ca30ea 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -1089,7 +1089,7 @@ void PPC64::writePltHeader(uint8_t *buf) const {
void PPC64::writePlt(uint8_t *buf, const Symbol &sym,
uint64_t /*pltEntryAddr*/) const {
- int32_t offset = pltHeaderSize + sym.pltIndex * pltEntrySize;
+ int32_t offset = pltHeaderSize + sym.getPltIdx() * pltEntrySize;
// bl __glink_PLTresolve
write32(buf, 0x48000000 | ((-offset) & 0x03FFFFFc));
}
diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp
index 2560dc8832575..e084bd6469128 100644
--- a/lld/ELF/Arch/X86.cpp
+++ b/lld/ELF/Arch/X86.cpp
@@ -220,7 +220,7 @@ void X86::writePltHeader(uint8_t *buf) const {
void X86::writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const {
- unsigned relOff = in.relaPlt->entsize * sym.pltIndex;
+ unsigned relOff = in.relaPlt->entsize * sym.getPltIdx();
if (config->isPic) {
const uint8_t inst[] = {
0xff, 0xa3, 0, 0, 0, 0, // jmp *foo at GOT(%ebx)
@@ -502,7 +502,7 @@ IntelIBT::IntelIBT() { pltHeaderSize = 0; }
void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const {
uint64_t va =
- in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize;
+ in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize;
write32le(buf, va);
}
@@ -600,7 +600,7 @@ void RetpolinePic::writePltHeader(uint8_t *buf) const {
void RetpolinePic::writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const {
- unsigned relOff = in.relaPlt->entsize * sym.pltIndex;
+ unsigned relOff = in.relaPlt->entsize * sym.getPltIdx();
const uint8_t insn[] = {
0x50, // pushl %eax
0x8b, 0x83, 0, 0, 0, 0, // mov foo at GOT(%ebx), %eax
@@ -659,7 +659,7 @@ void RetpolineNoPic::writePltHeader(uint8_t *buf) const {
void RetpolineNoPic::writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const {
- unsigned relOff = in.relaPlt->entsize * sym.pltIndex;
+ unsigned relOff = in.relaPlt->entsize * sym.getPltIdx();
const uint8_t insn[] = {
0x50, // 0: pushl %eax
0xa1, 0, 0, 0, 0, // 1: mov foo_in_GOT, %eax
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 161e99e3ba3fb..88ad8682e5b0e 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -417,7 +417,7 @@ void X86_64::writePlt(uint8_t *buf, const Symbol &sym,
memcpy(buf, inst, sizeof(inst));
write32le(buf + 2, sym.getGotPltVA() - pltEntryAddr - 6);
- write32le(buf + 7, sym.pltIndex);
+ write32le(buf + 7, sym.getPltIdx());
write32le(buf + 12, in.plt->getVA() - pltEntryAddr - 16);
}
@@ -995,7 +995,7 @@ IntelIBT::IntelIBT() { pltHeaderSize = 0; }
void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const {
uint64_t va =
- in.ibtPlt->getVA() + IBTPltHeaderSize + s.pltIndex * pltEntrySize;
+ in.ibtPlt->getVA() + IBTPltHeaderSize + s.getPltIdx() * pltEntrySize;
write64le(buf, va);
}
@@ -1107,7 +1107,7 @@ void Retpoline::writePlt(uint8_t *buf, const Symbol &sym,
write32le(buf + 3, sym.getGotPltVA() - pltEntryAddr - 7);
write32le(buf + 8, -off - 12 + 32);
write32le(buf + 13, -off - 17 + 18);
- write32le(buf + 18, sym.pltIndex);
+ write32le(buf + 18, sym.getPltIdx());
write32le(buf + 23, -off - 27);
}
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index e93f83906d522..e4c252a8988f1 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -96,6 +96,7 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
sharedFiles.clear();
backwardReferences.clear();
whyExtract.clear();
+ symAux.clear();
tar = nullptr;
in.reset();
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index ec19139699458..7c099700733f9 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -302,8 +302,7 @@ static void replaceWithDefined(Symbol &sym, SectionBase &sec, uint64_t value,
sym.replace(Defined{sym.file, sym.getName(), sym.binding, sym.stOther,
sym.type, value, size, &sec});
- sym.pltIndex = old.pltIndex;
- sym.gotIndex = old.gotIndex;
+ sym.auxIdx = old.auxIdx;
sym.verdefIndex = old.verdefIndex;
sym.exportDynamic = true;
sym.isUsedInRegularObj = true;
@@ -1544,15 +1543,17 @@ static bool handleNonPreemptibleIfunc(Symbol &sym) {
// may alter section/value, so create a copy of the symbol to make
// section/value fixed.
auto *directSym = makeDefined(cast<Defined>(sym));
+ directSym->allocateAux();
addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel,
*directSym);
- sym.pltIndex = directSym->pltIndex;
+ sym.allocateAux();
+ symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx;
if (sym.hasDirectReloc) {
// Change the value to the IPLT and redirect all references to it.
auto &d = cast<Defined>(sym);
d.section = in.iplt.get();
- d.value = sym.pltIndex * target->ipltEntrySize;
+ d.value = d.getPltIdx() * target->ipltEntrySize;
d.size = 0;
// It's important to set the symbol type here so that dynamic loaders
// don't try to call the PLT as if it were an ifunc resolver.
@@ -1571,6 +1572,10 @@ void elf::postScanRelocations() {
auto fn = [](Symbol &sym) {
if (handleNonPreemptibleIfunc(sym))
return;
+ if (!sym.needsDynReloc())
+ return;
+ sym.allocateAux();
+
if (sym.needsGot)
addGotEntry(sym);
if (sym.needsPlt)
@@ -1584,9 +1589,10 @@ void elf::postScanRelocations() {
} else {
assert(sym.isFunc() && sym.needsPlt);
if (!sym.isDefined()) {
- replaceWithDefined(
- sym, *in.plt,
- target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0);
+ replaceWithDefined(sym, *in.plt,
+ target->pltHeaderSize +
+ target->pltEntrySize * sym.getPltIdx(),
+ 0);
sym.needsCopy = true;
if (config->emachine == EM_PPC) {
// PPC32 canonical PLT entries are at the beginning of .glink
@@ -1653,6 +1659,8 @@ void elf::postScanRelocations() {
if (sym.needsTlsIe && !sym.needsTlsGdToIe)
addTpOffsetGotEntry(sym);
};
+
+ assert(symAux.empty());
for (Symbol *sym : symtab->symbols())
fn(*sym);
@@ -2173,6 +2181,7 @@ void elf::hexagonTLSSymbolUpdate(ArrayRef<OutputSection *> outputSections) {
for (Relocation &rel : isec->relocations)
if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) {
if (needEntry) {
+ sym->allocateAux();
addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel,
*sym);
needEntry = false;
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 443bec0c111fc..f3bd67b858737 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -59,6 +59,7 @@ DenseMap<const Symbol *, std::pair<const InputFile *, const InputFile *>>
elf::backwardReferences;
SmallVector<std::tuple<std::string, const InputFile *, const Symbol &>, 0>
elf::whyExtract;
+SmallVector<SymbolAux, 0> elf::symAux;
static uint64_t getSymVA(const Symbol &sym, int64_t addend) {
switch (sym.kind()) {
@@ -153,7 +154,7 @@ uint64_t Symbol::getGotVA() const {
}
uint64_t Symbol::getGotOffset() const {
- return gotIndex * target->gotEntrySize;
+ return getGotIdx() * target->gotEntrySize;
}
uint64_t Symbol::getGotPltVA() const {
@@ -164,15 +165,15 @@ uint64_t Symbol::getGotPltVA() const {
uint64_t Symbol::getGotPltOffset() const {
if (isInIplt)
- return pltIndex * target->gotEntrySize;
- return (pltIndex + target->gotPltHeaderEntriesNum) * target->gotEntrySize;
+ return getPltIdx() * target->gotEntrySize;
+ return (getPltIdx() + target->gotPltHeaderEntriesNum) * target->gotEntrySize;
}
uint64_t Symbol::getPltVA() const {
uint64_t outVA = isInIplt
- ? in.iplt->getVA() + pltIndex * target->ipltEntrySize
+ ? in.iplt->getVA() + getPltIdx() * target->ipltEntrySize
: in.plt->getVA() + in.plt->headerSize +
- pltIndex * target->pltEntrySize;
+ getPltIdx() * target->pltEntrySize;
// While linking microMIPS code PLT code are always microMIPS
// code. Set the less-significant bit to track that fact.
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index e2dc76f576af7..d9935c3a7c4b6 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -56,6 +56,16 @@ struct StringRefZ {
const uint32_t size;
};
+// Some index properties of a symbol are stored separately in this auxiliary
+// struct to decrease sizeof(SymbolUnion) in the majority of cases.
+struct SymbolAux {
+ uint32_t gotIdx = -1;
+ uint32_t pltIdx = -1;
+ uint32_t tlsGdIdx = -1;
+};
+
+extern SmallVector<SymbolAux, 0> symAux;
+
// The base class for real symbol classes.
class Symbol {
public:
@@ -79,11 +89,10 @@ class Symbol {
mutable uint32_t nameSize;
public:
+ // A symAux index used to access GOT/PLT entry indexes. This is allocated in
+ // postScanRelocations().
+ uint32_t auxIdx = -1;
uint32_t dynsymIndex = 0;
- uint32_t gotIndex = -1;
- uint32_t pltIndex = -1;
-
- uint32_t globalDynIndex = -1;
// This field is a index to the symbol's version definition.
uint16_t verdefIndex = -1;
@@ -191,8 +200,18 @@ class Symbol {
return nameData + nameSize;
}
- bool isInGot() const { return gotIndex != -1U; }
- bool isInPlt() const { return pltIndex != -1U; }
+ uint32_t getGotIdx() const {
+ return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].gotIdx;
+ }
+ uint32_t getPltIdx() const {
+ return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].pltIdx;
+ }
+ uint32_t getTlsGdIdx() const {
+ return auxIdx == uint32_t(-1) ? uint32_t(-1) : symAux[auxIdx].tlsGdIdx;
+ }
+
+ bool isInGot() const { return getGotIdx() != uint32_t(-1); }
+ bool isInPlt() const { return getPltIdx() != uint32_t(-1); }
uint64_t getVA(int64_t addend = 0) const;
@@ -301,6 +320,16 @@ class Symbol {
uint8_t needsTlsIe : 1;
uint8_t hasDirectReloc : 1;
+ bool needsDynReloc() const {
+ return needsCopy || needsGot || needsPlt || needsTlsDesc || needsTlsGd ||
+ needsTlsGdToIe || needsTlsLd || needsGotDtprel || needsTlsIe;
+ }
+ void allocateAux() {
+ assert(auxIdx == uint32_t(-1));
+ auxIdx = symAux.size();
+ symAux.emplace_back();
+ }
+
// The partition whose dynamic symbol table contains this symbol's definition.
uint8_t partition = 1;
@@ -504,9 +533,9 @@ union SymbolUnion {
};
// It is important to keep the size of SymbolUnion small for performance and
-// memory usage reasons. 80 bytes is a soft limit based on the size of Defined
+// memory usage reasons. 72 bytes is a soft limit based on the size of Defined
// on a 64-bit system.
-static_assert(sizeof(SymbolUnion) <= 80, "SymbolUnion too large");
+static_assert(sizeof(SymbolUnion) <= 72, "SymbolUnion too large");
template <typename T> struct AssertSymbol {
static_assert(std::is_trivially_destructible<T>(),
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index f93c09b95d751..6af10237fb83a 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -650,14 +650,13 @@ GotSection::GotSection()
}
void GotSection::addEntry(Symbol &sym) {
- sym.gotIndex = numEntries;
- ++numEntries;
+ assert(sym.auxIdx == symAux.size() - 1);
+ symAux.back().gotIdx = numEntries++;
}
bool GotSection::addDynTlsEntry(Symbol &sym) {
- if (sym.globalDynIndex != -1U)
- return false;
- sym.globalDynIndex = numEntries;
+ assert(sym.auxIdx == symAux.size() - 1);
+ symAux.back().tlsGdIdx = numEntries;
// Global Dynamic TLS entries take two GOT slots.
numEntries += 2;
return true;
@@ -674,11 +673,11 @@ bool GotSection::addTlsIndex() {
}
uint64_t GotSection::getGlobalDynAddr(const Symbol &b) const {
- return this->getVA() + b.globalDynIndex * config->wordsize;
+ return this->getVA() + b.getTlsGdIdx() * config->wordsize;
}
uint64_t GotSection::getGlobalDynOffset(const Symbol &b) const {
- return b.globalDynIndex * config->wordsize;
+ return b.getTlsGdIdx() * config->wordsize;
}
void GotSection::finalizeContents() {
@@ -972,12 +971,18 @@ void MipsGotSection::build() {
}
}
- // Update Symbol::gotIndex field to use this
+ // Update SymbolAux::gotIdx field to use this
// value later in the `sortMipsSymbols` function.
- for (auto &p : primGot->global)
- p.first->gotIndex = p.second;
- for (auto &p : primGot->relocs)
- p.first->gotIndex = p.second;
+ for (auto &p : primGot->global) {
+ if (p.first->auxIdx == uint32_t(-1))
+ p.first->allocateAux();
+ symAux.back().gotIdx = p.second;
+ }
+ for (auto &p : primGot->relocs) {
+ if (p.first->auxIdx == uint32_t(-1))
+ p.first->allocateAux();
+ symAux.back().gotIdx = p.second;
+ }
// Create dynamic relocations.
for (FileGot &got : gots) {
@@ -1145,7 +1150,8 @@ GotPltSection::GotPltSection()
}
void GotPltSection::addEntry(Symbol &sym) {
- assert(sym.pltIndex == entries.size());
+ assert(sym.auxIdx == symAux.size() - 1 &&
+ symAux.back().pltIdx == entries.size());
entries.push_back(&sym);
}
@@ -1190,7 +1196,7 @@ IgotPltSection::IgotPltSection()
target->gotEntrySize, getIgotPltName()) {}
void IgotPltSection::addEntry(Symbol &sym) {
- assert(sym.pltIndex == entries.size());
+ assert(symAux.back().pltIdx == entries.size());
entries.push_back(&sym);
}
@@ -2069,7 +2075,7 @@ static bool sortMipsSymbols(const SymbolTableEntry &l,
// Sort entries related to non-local preemptible symbols by GOT indexes.
// All other entries go to the beginning of a dynsym in arbitrary order.
if (l.sym->isInGot() && r.sym->isInGot())
- return l.sym->gotIndex < r.sym->gotIndex;
+ return l.sym->getGotIdx() < r.sym->getGotIdx();
if (!l.sym->isInGot() && !r.sym->isInGot())
return false;
return !l.sym->isInGot();
@@ -2546,7 +2552,8 @@ void PltSection::writeTo(uint8_t *buf) {
}
void PltSection::addEntry(Symbol &sym) {
- sym.pltIndex = entries.size();
+ assert(sym.auxIdx == symAux.size() - 1);
+ symAux.back().pltIdx = entries.size();
entries.push_back(&sym);
}
@@ -2592,7 +2599,8 @@ size_t IpltSection::getSize() const {
}
void IpltSection::addEntry(Symbol &sym) {
- sym.pltIndex = entries.size();
+ assert(sym.auxIdx == symAux.size() - 1);
+ symAux.back().pltIdx = entries.size();
entries.push_back(&sym);
}
More information about the llvm-commits
mailing list