[lld] [ELF] Replace getThreadIndex with explicit shards in RelocScan. NFC (PR #208959)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 11 15:26:22 PDT 2026
https://github.com/MaskRay created https://github.com/llvm/llvm-project/pull/208959
Parallel relocation scanning (https://reviews.llvm.org/D133003) appends
dynamic relocations to `relocsVec[parallel::getThreadIndex()]`.
`getThreadIndex` returns -1u on main, which is a known hazard.
Instead, run one scan task per worker via `parallelFor`, with tasks
claiming object files off a shared counter, and pass the task index
through scanSection/RelocScan as the relocsVec shard.
>From deeb014b7545c709eee8f35987bc5156ca86c98b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 11 Jul 2026 13:15:32 -0700
Subject: [PATCH] [ELF] Replace getThreadIndex with explicit shards in
RelocScan. NFC
Parallel relocation scanning (https://reviews.llvm.org/D133003) appends
dynamic relocations to `relocsVec[parallel::getThreadIndex()]`.
`getThreadIndex` returns -1u on main, which is a known hazard.
Instead, run one scan task per worker via `parallelFor`, with tasks
claiming object files off a shared counter, and pass the task index
through scanSection/RelocScan as the relocsVec shard.
---
lld/ELF/Arch/AArch64.cpp | 14 ++++---
lld/ELF/Arch/ARM.cpp | 14 ++++---
lld/ELF/Arch/Hexagon.cpp | 12 +++---
lld/ELF/Arch/LoongArch.cpp | 13 +++---
lld/ELF/Arch/Mips.cpp | 16 ++++----
lld/ELF/Arch/PPC.cpp | 15 +++----
lld/ELF/Arch/PPC64.cpp | 15 +++----
lld/ELF/Arch/RISCV.cpp | 15 +++----
lld/ELF/Arch/SPARCV9.cpp | 12 +++---
lld/ELF/Arch/SystemZ.cpp | 14 ++++---
lld/ELF/Arch/X86.cpp | 14 ++++---
lld/ELF/Arch/X86_64.cpp | 16 ++++----
lld/ELF/RelocScan.h | 18 ++++----
lld/ELF/Relocations.cpp | 82 +++++++++++++++++--------------------
lld/ELF/SyntheticSections.h | 48 ++++++++--------------
lld/ELF/Target.h | 9 ++--
16 files changed, 167 insertions(+), 160 deletions(-)
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 11c3dca1a358a..4d24c49382559 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -80,12 +80,13 @@ class AArch64 : public TargetInfo {
void writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override {
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override {
if (ctx.arg.ekind == ELF64BEKind)
- elf::scanSection1<AArch64, ELF64BE>(*this, sec);
+ elf::scanSection1<AArch64, ELF64BE>(*this, sec, shard);
else
- elf::scanSection1<AArch64, ELF64LE>(*this, sec);
+ elf::scanSection1<AArch64, ELF64LE>(*this, sec, shard);
}
bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
uint64_t branchAddr, const Symbol &s,
@@ -192,8 +193,9 @@ bool AArch64::usesOnlyLowPageBits(RelType type) const {
}
template <class ELFT, class RelTy>
-void AArch64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void AArch64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index e0b2f321264b5..e1f73b991f1f3 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -55,12 +55,13 @@ class ARM final : public TargetInfo {
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override {
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override {
if (ctx.arg.ekind == ELF32BEKind)
- elf::scanSection1<ARM, ELF32BE>(*this, sec);
+ elf::scanSection1<ARM, ELF32BE>(*this, sec, shard);
else
- elf::scanSection1<ARM, ELF32LE>(*this, sec);
+ elf::scanSection1<ARM, ELF32LE>(*this, sec, shard);
}
DenseMap<InputSection *, SmallVector<const Defined *, 0>> sectionMap;
@@ -169,8 +170,9 @@ RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
}
template <class ELFT, class RelTy>
-void ARM::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void ARM::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
const RelTy &rel = *it;
diff --git a/lld/ELF/Arch/Hexagon.cpp b/lld/ELF/Arch/Hexagon.cpp
index e136eda174b15..985f9c365bd37 100644
--- a/lld/ELF/Arch/Hexagon.cpp
+++ b/lld/ELF/Arch/Hexagon.cpp
@@ -40,9 +40,10 @@ class Hexagon final : public TargetInfo {
RelType getDynRel(RelType type) const override;
int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override {
- elf::scanSection1<Hexagon, ELF32LE>(*this, sec);
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override {
+ elf::scanSection1<Hexagon, ELF32LE>(*this, sec, shard);
}
void finalizeRelocScan() override;
bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
@@ -127,8 +128,9 @@ RelExpr Hexagon::getRelExpr(RelType type, const Symbol &s,
}
template <class ELFT, class RelTy>
-void Hexagon::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void Hexagon::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
const RelTy &rel = *it;
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index 3302da394c931..224c31d5a2f1d 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -38,12 +38,12 @@ class LoongArch final : public TargetInfo {
const uint8_t *loc) const override;
bool usesOnlyLowPageBits(RelType type) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
- void scanSection(InputSectionBase &sec) override {
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override {
if (ctx.arg.is64)
- elf::scanSection1<LoongArch, ELF64LE>(*this, sec);
+ elf::scanSection1<LoongArch, ELF64LE>(*this, sec, shard);
else
- elf::scanSection1<LoongArch, ELF32LE>(*this, sec);
+ elf::scanSection1<LoongArch, ELF32LE>(*this, sec, shard);
}
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
@@ -479,8 +479,9 @@ bool LoongArch::usesOnlyLowPageBits(RelType type) const {
}
template <class ELFT, class RelTy>
-void LoongArch::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void LoongArch::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
RelType type = it->getType(false);
diff --git a/lld/ELF/Arch/Mips.cpp b/lld/ELF/Arch/Mips.cpp
index d8520208a12c0..ba751660df1fa 100644
--- a/lld/ELF/Arch/Mips.cpp
+++ b/lld/ELF/Arch/Mips.cpp
@@ -34,8 +34,8 @@ template <class ELFT> class MIPS final : public TargetInfo {
void writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
- void scanSection(InputSectionBase &) override;
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
+ void scanSection(InputSectionBase &, unsigned shard) override;
bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
uint64_t branchAddr, const Symbol &s,
int64_t a) const override;
@@ -670,8 +670,9 @@ static RelType getMipsPairType(RelType type, bool isLocal) {
template <class ELFT>
template <class RelTy>
-void MIPS<ELFT>::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void MIPS<ELFT>::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
RelType type;
for (auto it = rels.begin(); it != rels.end();) {
@@ -741,12 +742,13 @@ void MIPS<ELFT>::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-template <class ELFT> void MIPS<ELFT>::scanSection(InputSectionBase &sec) {
+template <class ELFT>
+void MIPS<ELFT>::scanSection(InputSectionBase &sec, unsigned shard) {
auto relocs = sec.template relsOrRelas<ELFT>();
if (relocs.areRelocsRel())
- scanSectionImpl(sec, relocs.rels);
+ scanSectionImpl(sec, relocs.rels, shard);
else
- scanSectionImpl(sec, relocs.relas);
+ scanSectionImpl(sec, relocs.relas, shard);
}
template <class ELFT>
diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index fa64271e12380..fdd1351f205bb 100644
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -44,8 +44,8 @@ class PPC final : public TargetInfo {
uint64_t pltEntryAddr) const override;
void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
- void scanSection(InputSectionBase &) override;
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
+ void scanSection(InputSectionBase &, unsigned shard) override;
bool needsThunk(RelExpr expr, RelType relocType, const InputFile *file,
uint64_t branchAddr, const Symbol &s,
int64_t a) const override;
@@ -285,8 +285,9 @@ int64_t PPC::getImplicitAddend(const uint8_t *buf, RelType type) const {
}
template <class ELFT, class RelTy>
-void PPC::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void PPC::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
const RelTy &rel = *it;
@@ -405,11 +406,11 @@ void PPC::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-void PPC::scanSection(InputSectionBase &sec) {
+void PPC::scanSection(InputSectionBase &sec, unsigned shard) {
if (ctx.arg.isLE)
- elf::scanSection1<PPC, ELF32LE>(*this, sec);
+ elf::scanSection1<PPC, ELF32LE>(*this, sec, shard);
else
- elf::scanSection1<PPC, ELF32BE>(*this, sec);
+ elf::scanSection1<PPC, ELF32BE>(*this, sec, shard);
}
static std::pair<RelType, uint64_t> fromDTPREL(RelType type, uint64_t val) {
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 9f49605d89c58..0cbf8acde96e0 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -180,8 +180,8 @@ class PPC64 final : public TargetInfo {
void writeIplt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
- void scanSection(InputSectionBase &) override;
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
+ void scanSection(InputSectionBase &, unsigned shard) override;
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
void writeGotHeader(uint8_t *buf) const override;
@@ -1114,8 +1114,9 @@ static bool missingTlsGdLdMarker(InputSectionBase &sec, Relocs<RelTy> rels) {
}
template <class ELFT, class RelTy>
-void PPC64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void PPC64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
bool optimizeTlsGdLd =
!missingTlsGdLdMarker<RelTy>(sec, rels) && !ctx.arg.shared;
@@ -1352,11 +1353,11 @@ void PPC64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-void PPC64::scanSection(InputSectionBase &sec) {
+void PPC64::scanSection(InputSectionBase &sec, unsigned shard) {
if (ctx.arg.isLE)
- elf::scanSection1<PPC64, ELF64LE>(*this, sec);
+ elf::scanSection1<PPC64, ELF64LE>(*this, sec, shard);
else
- elf::scanSection1<PPC64, ELF64BE>(*this, sec);
+ elf::scanSection1<PPC64, ELF64BE>(*this, sec, shard);
// Sort relocations by offset for .toc sections. This is needed so that
// sections addressed with small code model relocations come first.
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 576e7a4614572..b88136326f449 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -40,8 +40,8 @@ class RISCV final : public TargetInfo {
void writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
- void scanSection(InputSectionBase &) override;
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
+ void scanSection(InputSectionBase &, unsigned shard) override;
RelType getDynRel(RelType type) const override;
RelExpr getRelExpr(RelType type, const Symbol &s,
const uint8_t *loc) const override;
@@ -315,8 +315,9 @@ RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
}
template <class ELFT, class RelTy>
-void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
// Many relocations end up in sec.relocations.
sec.relocations.reserve(rels.size());
@@ -476,11 +477,11 @@ void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
});
}
-void RISCV::scanSection(InputSectionBase &sec) {
+void RISCV::scanSection(InputSectionBase &sec, unsigned shard) {
if (ctx.arg.is64)
- elf::scanSection1<RISCV, ELF64LE>(*this, sec);
+ elf::scanSection1<RISCV, ELF64LE>(*this, sec, shard);
else
- elf::scanSection1<RISCV, ELF32LE>(*this, sec);
+ elf::scanSection1<RISCV, ELF32LE>(*this, sec, shard);
}
void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
diff --git a/lld/ELF/Arch/SPARCV9.cpp b/lld/ELF/Arch/SPARCV9.cpp
index 405be69a8f572..fad83da0d397d 100644
--- a/lld/ELF/Arch/SPARCV9.cpp
+++ b/lld/ELF/Arch/SPARCV9.cpp
@@ -28,9 +28,10 @@ class SPARCV9 final : public TargetInfo {
void writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override {
- elf::scanSection1<SPARCV9, ELF64BE>(*this, sec);
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override {
+ elf::scanSection1<SPARCV9, ELF64BE>(*this, sec, shard);
}
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
@@ -73,8 +74,9 @@ RelExpr SPARCV9::getRelExpr(RelType type, const Symbol &s,
}
template <class ELFT, class RelTy>
-void SPARCV9::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void SPARCV9::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
const RelTy &rel = *it;
diff --git a/lld/ELF/Arch/SystemZ.cpp b/lld/ELF/Arch/SystemZ.cpp
index 545380c2c8763..b76baf464a5e7 100644
--- a/lld/ELF/Arch/SystemZ.cpp
+++ b/lld/ELF/Arch/SystemZ.cpp
@@ -35,8 +35,9 @@ class SystemZ : public TargetInfo {
void writePlt(uint8_t *buf, const Symbol &sym,
uint64_t pltEntryAddr) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override;
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override;
RelExpr adjustGotPcExpr(RelType type, int64_t addend,
const uint8_t *loc) const override;
bool relaxOnce(int pass) const override;
@@ -192,8 +193,9 @@ int64_t SystemZ::getImplicitAddend(const uint8_t *buf, RelType type) const {
}
template <class ELFT, class RelTy>
-void SystemZ::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void SystemZ::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
@@ -359,8 +361,8 @@ void SystemZ::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-void SystemZ::scanSection(InputSectionBase &sec) {
- elf::scanSection1<SystemZ, ELF64BE>(*this, sec);
+void SystemZ::scanSection(InputSectionBase &sec, unsigned shard) {
+ elf::scanSection1<SystemZ, ELF64BE>(*this, sec, shard);
}
RelType SystemZ::getDynRel(RelType type) const {
diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp
index 1f9c247511c72..5b95ccdf50e46 100644
--- a/lld/ELF/Arch/X86.cpp
+++ b/lld/ELF/Arch/X86.cpp
@@ -37,8 +37,9 @@ class X86 : public TargetInfo {
void relocate(uint8_t *loc, const Relocation &rel,
uint64_t val) const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override;
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override;
void relocateAlloc(InputSection &sec, uint8_t *buf) const override;
private:
@@ -176,8 +177,9 @@ void X86::writePlt(uint8_t *buf, const Symbol &sym,
}
template <class ELFT, class RelTy>
-void X86::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void X86::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
@@ -291,8 +293,8 @@ void X86::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-void X86::scanSection(InputSectionBase &sec) {
- elf::scanSection1<X86, ELF32LE>(*this, sec);
+void X86::scanSection(InputSectionBase &sec, unsigned shard) {
+ elf::scanSection1<X86, ELF32LE>(*this, sec, shard);
}
int64_t X86::getImplicitAddend(const uint8_t *buf, RelType type) const {
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 1618320178954..fee33b03ffa6a 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -54,8 +54,9 @@ class X86_64 : public TargetInfo {
void relaxCFIJumpTables() const override;
void applyBranchToBranchOpt() const override;
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
- void scanSection(InputSectionBase &sec) override;
+ void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard);
+ void scanSection(InputSectionBase &sec, unsigned shard) override;
private:
void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
@@ -642,8 +643,9 @@ RelType X86_64::getDynRel(RelType type) const {
}
template <class ELFT, class RelTy>
-void X86_64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void X86_64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
sec.relocations.reserve(rels.size());
for (auto it = rels.begin(); it != rels.end(); ++it) {
@@ -769,11 +771,11 @@ void X86_64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
[](auto &l, auto &r) { return l.offset < r.offset; });
}
-void X86_64::scanSection(InputSectionBase &sec) {
+void X86_64::scanSection(InputSectionBase &sec, unsigned shard) {
if (ctx.arg.is64)
- elf::scanSection1<X86_64, ELF64LE>(*this, sec);
+ elf::scanSection1<X86_64, ELF64LE>(*this, sec, shard);
else // ilp32
- elf::scanSection1<X86_64, ELF32LE>(*this, sec);
+ elf::scanSection1<X86_64, ELF32LE>(*this, sec, shard);
}
void X86_64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
diff --git a/lld/ELF/RelocScan.h b/lld/ELF/RelocScan.h
index c0292d35517fa..c39089840d2fb 100644
--- a/lld/ELF/RelocScan.h
+++ b/lld/ELF/RelocScan.h
@@ -51,8 +51,11 @@ class RelocScan {
public:
Ctx &ctx;
InputSectionBase *sec;
+ // `relocsVec` shard that discovered dynamic relocations are appended to.
+ unsigned shard;
- RelocScan(Ctx &ctx, InputSectionBase *sec = nullptr) : ctx(ctx), sec(sec) {}
+ RelocScan(Ctx &ctx, InputSectionBase *sec, unsigned shard)
+ : ctx(ctx), sec(sec), shard(shard) {}
template <class ELFT, class RelTy>
void scan(typename Relocs<RelTy>::const_iterator &i, RelType type,
int64_t addend);
@@ -113,8 +116,9 @@ class RelocScan {
// PIC when the relocation uses the full address (not just low page bits).
if (ieExpr == R_GOT && ctx.arg.isPic &&
!ctx.target->usesOnlyLowPageBits(type))
- ctx.in.relaDyn->addRelativeReloc<true>(
- ctx.target->relativeRel, *sec, offset, sym, addend, type, ieExpr);
+ ctx.in.relaDyn->addRelativeReloc<true>(ctx.target->relativeRel, *sec,
+ offset, sym, addend, type,
+ ieExpr, shard);
else
sec->addReloc({ieExpr, type, offset, addend, &sym});
}
@@ -202,14 +206,14 @@ void RelocScan::scan(typename Relocs<RelTy>::const_iterator &it, RelType type,
// Dispatch to target-specific scanSectionImpl based on relocation format.
template <class Target, class ELFT>
-void scanSection1(Target &target, InputSectionBase &sec) {
+void scanSection1(Target &target, InputSectionBase &sec, unsigned shard) {
const RelsOrRelas<ELFT> rels = sec.template relsOrRelas<ELFT>();
if (rels.areRelocsCrel())
- target.template scanSectionImpl<ELFT>(sec, rels.crels);
+ target.template scanSectionImpl<ELFT>(sec, rels.crels, shard);
else if (rels.areRelocsRel())
- target.template scanSectionImpl<ELFT>(sec, rels.rels);
+ target.template scanSectionImpl<ELFT>(sec, rels.rels, shard);
else
- target.template scanSectionImpl<ELFT>(sec, rels.relas);
+ target.template scanSectionImpl<ELFT>(sec, rels.relas, shard);
}
} // namespace lld::elf
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index bea8fa681c60e..4702d941d28ca 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -41,7 +41,9 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Demangle/Demangle.h"
+#include "llvm/Support/Parallel.h"
#include <algorithm>
+#include <atomic>
using namespace llvm;
using namespace llvm::ELF;
@@ -697,10 +699,10 @@ bool RelocScan::checkTlsLe(uint64_t offset, Symbol &sym, RelType type) {
return true;
}
-template <bool shard = false>
+template <bool concurrent = false>
static void addRelativeReloc(Ctx &ctx, InputSectionBase &isec,
uint64_t offsetInSec, Symbol &sym, int64_t addend,
- RelExpr expr, RelType type) {
+ RelExpr expr, RelType type, unsigned shard = 0) {
bool isAArch64Auth =
ctx.arg.emachine == EM_AARCH64 && type == R_AARCH64_AUTH_ABS64;
@@ -722,15 +724,15 @@ static void addRelativeReloc(Ctx &ctx, InputSectionBase &isec,
if (sym.isTagged())
relrDyn = nullptr;
if (relrDyn && isec.addralign >= 2 && offsetInSec % 2 == 0) {
- relrDyn->addRelativeReloc<shard>(isec, offsetInSec, sym, addend, type,
- expr);
+ relrDyn->addRelativeReloc(isec, offsetInSec, sym, addend, type, expr,
+ shard);
return;
}
RelType relativeType = ctx.target->relativeRel;
if (isAArch64Auth)
relativeType = R_AARCH64_AUTH_RELATIVE;
- ctx.in.relaDyn->addRelativeReloc<shard>(relativeType, isec, offsetInSec, sym,
- addend, type, expr);
+ ctx.in.relaDyn->addRelativeReloc<concurrent>(relativeType, isec, offsetInSec,
+ sym, addend, type, expr, shard);
// With MTE globals, we always want to derive the address tag by `ldg`-ing
// the symbol. When we have a RELATIVE relocation though, we no longer have
// a reference to the symbol. Because of this, when we have an addend that
@@ -995,7 +997,7 @@ void RelocScan::processAux(RelExpr expr, RelType type, uint64_t offset,
((rel == ctx.target->symbolicRel ||
(ctx.arg.emachine == EM_AARCH64 && type == R_AARCH64_AUTH_ABS64)) &&
!sym.isPreemptible)) {
- addRelativeReloc<true>(ctx, *sec, offset, sym, addend, expr, type);
+ addRelativeReloc<true>(ctx, *sec, offset, sym, addend, expr, type, shard);
return;
}
if (rel != 0) {
@@ -1121,8 +1123,9 @@ void RelocScan::processAux(RelExpr expr, RelType type, uint64_t offset,
}
template <class ELFT, class RelTy>
-void TargetInfo::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
- RelocScan rs(ctx, &sec);
+void TargetInfo::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
+ unsigned shard) {
+ RelocScan rs(ctx, &sec, shard);
// Many relocations end up in sec.relocations.
sec.relocations.reserve(rels.size());
@@ -1132,18 +1135,19 @@ void TargetInfo::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
}
}
-template <class ELFT> void TargetInfo::scanSection1(InputSectionBase &sec) {
+template <class ELFT>
+void TargetInfo::scanSection1(InputSectionBase &sec, unsigned shard) {
const RelsOrRelas<ELFT> rels = sec.template relsOrRelas<ELFT>();
if (rels.areRelocsCrel())
- scanSectionImpl<ELFT>(sec, rels.crels);
+ scanSectionImpl<ELFT>(sec, rels.crels, shard);
else if (rels.areRelocsRel())
- scanSectionImpl<ELFT>(sec, rels.rels);
+ scanSectionImpl<ELFT>(sec, rels.rels, shard);
else
- scanSectionImpl<ELFT>(sec, rels.relas);
+ scanSectionImpl<ELFT>(sec, rels.relas, shard);
}
-void TargetInfo::scanSection(InputSectionBase &sec) {
- invokeELFT(scanSection1, sec);
+void TargetInfo::scanSection(InputSectionBase &sec, unsigned shard) {
+ invokeELFT(scanSection1, sec, shard);
}
void RelocScan::scanEhSection(EhInputSection &s) {
@@ -1173,46 +1177,36 @@ template <class ELFT> void elf::scanRelocations(Ctx &ctx) {
// copy relocations, etc. Note that relocations for non-alloc sections are
// directly processed by InputSection::relocateNonAlloc.
+ size_t numFiles = ctx.objectFiles.size();
+ std::atomic<size_t> next{0};
// MIPS modifies MipsGotSection during relocation scanning, which is not
// suitable for parallelism.
- bool serial = ctx.arg.emachine == EM_MIPS;
- parallel::TaskGroup tg;
- auto outerFn = [&]() {
- for (ELFFileBase *f : ctx.objectFiles) {
- auto fn = [f, &ctx]() {
- for (InputSectionBase *s : f->getSections()) {
+ size_t numWorkers = ctx.arg.emachine == EM_MIPS
+ ? 1
+ : std::min<size_t>(ctx.arg.threadCount, numFiles + 1);
+ parallelFor(0, numWorkers, [&](unsigned shard) {
+ // Tasks claim work items off a shared counter: item i < numFiles scans
+ // ctx.objectFiles[i] while the last item scans special sections.
+ for (size_t i;
+ (i = next.fetch_add(1, std::memory_order_relaxed)) <= numFiles;) {
+ if (i != numFiles) {
+ for (InputSectionBase *s : ctx.objectFiles[i]->getSections())
if (s && s->kind() == SectionBase::Regular && s->isLive() &&
(s->flags & SHF_ALLOC) &&
!(s->type == SHT_ARM_EXIDX && ctx.arg.emachine == EM_ARM))
- ctx.target->scanSection(*s);
- }
- };
- if (serial)
- fn();
- else
- tg.spawn(fn);
- }
- auto scanEH = [&] {
- RelocScan scanner(ctx);
+ ctx.target->scanSection(*s, shard);
+ continue;
+ }
+ RelocScan scanner(ctx, nullptr, shard);
for (EhInputSection *sec : ctx.in.ehFrame->sections)
scanner.scanEhSection(*sec);
ARMExidxSyntheticSection *armExidx = ctx.in.armExidx.get();
if (armExidx && armExidx->isLive())
for (InputSection *sec : armExidx->exidxSections)
if (sec->isLive())
- ctx.target->scanSection(*sec);
- };
- if (serial)
- scanEH();
- else
- tg.spawn(scanEH);
- };
- // If `serial` is true, call `spawn` to ensure that `scanner` runs in a thread
- // with valid getThreadIndex().
- if (serial)
- tg.spawn(outerFn);
- else
- outerFn();
+ ctx.target->scanSection(*sec, shard);
+ }
+ });
}
RelocationBaseSection &elf::getIRelativeSection(Ctx &ctx) {
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 2b937558ea01c..523f6587899fe 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -34,7 +34,6 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Endian.h"
-#include "llvm/Support/Parallel.h"
#include "llvm/Support/Threading.h"
namespace lld::elf {
@@ -494,8 +493,12 @@ class RelocationBaseSection : public SyntheticSection {
/// Add a dynamic relocation without writing an addend to the output section.
/// This overload can be used if the addends are written directly instead of
/// using relocations on the input section (e.g. MipsGotSection::writeTo()).
- template <bool shard = false> void addReloc(const DynamicReloc &reloc) {
- if (reloc.type == relativeRel)
+ /// Concurrent callers must pass distinct shards.
+ template <bool concurrent = false>
+ void addReloc(const DynamicReloc &reloc, unsigned shard = 0) {
+ if constexpr (concurrent)
+ relocsVec[shard].push_back(reloc);
+ else if (reloc.type == relativeRel)
relativeRelocs.push_back(reloc);
else
relocs.push_back(reloc);
@@ -509,13 +512,14 @@ class RelocationBaseSection : public SyntheticSection {
/// This function should only be called for non-preemptible symbols or
/// RelExpr values that refer to an address inside the output file (e.g. the
/// address of the GOT entry for a potentially preemptible symbol).
- template <bool shard = false>
+ template <bool concurrent = false>
void addRelativeReloc(RelType dynType, InputSectionBase &isec,
uint64_t offsetInSec, Symbol &sym, int64_t addend,
- RelType addendRelType, RelExpr expr) {
+ RelType addendRelType, RelExpr expr,
+ unsigned shard = 0) {
assert(expr != R_ADDEND && "expected non-addend relocation expression");
- addReloc<shard>(false, dynType, isec, offsetInSec, sym, addend, expr,
- addendRelType);
+ addReloc<concurrent>(false, dynType, isec, offsetInSec, sym, addend, expr,
+ addendRelType, shard);
}
/// Add a dynamic relocation using the target address of \p sym as the addend
/// if \p sym is non-preemptible. Otherwise add a relocation against \p sym.
@@ -523,16 +527,17 @@ class RelocationBaseSection : public SyntheticSection {
InputSectionBase &isec,
uint64_t offsetInSec, Symbol &sym,
RelType addendRelType);
- template <bool shard = false>
+ template <bool concurrent = false>
void addReloc(bool isAgainstSymbol, RelType dynType, InputSectionBase &sec,
uint64_t offsetInSec, Symbol &sym, int64_t addend, RelExpr expr,
- RelType addendRelType) {
+ RelType addendRelType, unsigned shard = 0) {
// Write the addends to the relocated address if required. We skip
// it if the written value would be zero.
if (ctx.arg.writeAddends && (expr != R_ADDEND || addend != 0))
sec.addReloc({expr, addendRelType, offsetInSec, addend, &sym});
- addReloc<shard>(
- {dynType, &sec, offsetInSec, isAgainstSymbol, sym, addend, expr});
+ addReloc<concurrent>(
+ {dynType, &sec, offsetInSec, isAgainstSymbol, sym, addend, expr},
+ shard);
}
bool isNeeded() const override {
return !relocs.empty() || !relativeRelocs.empty() ||
@@ -561,11 +566,6 @@ class RelocationBaseSection : public SyntheticSection {
bool combreloc;
};
-template <>
-inline void RelocationBaseSection::addReloc<true>(const DynamicReloc &reloc) {
- relocsVec[llvm::parallel::getThreadIndex()].push_back(reloc);
-}
-
template <class ELFT>
class RelocationSection final : public RelocationBaseSection {
using Elf_Rel = typename ELFT::Rel;
@@ -607,21 +607,14 @@ struct RelativeReloc {
class RelrBaseSection : public SyntheticSection {
public:
RelrBaseSection(Ctx &, unsigned concurrency, bool isAArch64Auth = false);
- /// Add a dynamic relocation without writing an addend to the output section.
- /// This overload can be used if the addends are written directly instead of
- /// using relocations on the input section.
- template <bool shard = false> void addReloc(const RelativeReloc &reloc) {
- relocs.push_back(reloc);
- }
/// Add a relative dynamic relocation that uses the target address of \p sym
/// (i.e. InputSection::getRelocTargetVA()) + \p addend as the addend.
- template <bool shard = false>
void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec,
Symbol &sym, int64_t addend, RelType addendRelType,
- RelExpr expr) {
+ RelExpr expr, unsigned shard) {
assert(expr != R_ADDEND && "expected non-addend relocation expression");
isec.addReloc({expr, addendRelType, offsetInSec, addend, &sym});
- addReloc<shard>({&isec, isec.relocs().size() - 1});
+ relocsVec[shard].push_back({&isec, isec.relocs().size() - 1});
}
bool isNeeded() const override {
return !relocs.empty() ||
@@ -635,11 +628,6 @@ class RelrBaseSection : public SyntheticSection {
SmallVector<SmallVector<RelativeReloc, 0>, 0> relocsVec;
};
-template <>
-inline void RelrBaseSection::addReloc<true>(const RelativeReloc &reloc) {
- relocsVec[llvm::parallel::getThreadIndex()].push_back(reloc);
-}
-
// RelrSection is used to encode offsets for relative relocations.
// Proposal for adding SHT_RELR sections to generic-abi is here:
// https://groups.google.com/forum/#!topic/generic-abi/bX460iggiKg
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 199f76e3fe851..50f0cd8a92161 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -90,13 +90,14 @@ class TargetInfo {
uint64_t dst) const;
// Function for scanning relocation. Typically overridden by targets that
- // require special type or addend adjustment.
- virtual void scanSection(InputSectionBase &);
+ // require special type or addend adjustment. `shard` selects the `relocsVec`
+ // shard that discovered dynamic relocations are appended to.
+ virtual void scanSection(InputSectionBase &, unsigned shard);
// Called by scanSection as a default implementation for specific ELF
// relocation types.
- template <class ELFT> void scanSection1(InputSectionBase &);
+ template <class ELFT> void scanSection1(InputSectionBase &, unsigned shard);
template <class ELFT, class RelTy>
- void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
+ void scanSectionImpl(InputSectionBase &, Relocs<RelTy>, unsigned shard);
// Called after parallel relocation scanning is complete but before
// postScanRelocations processes symbol flags. Targets may override this to
More information about the llvm-commits
mailing list