[llvm-branch-commits] [lld] [lld][MachO] Batch command-line object files and archive member scanning in parallel (PR #240984)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 10 12:05:18 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-lld
@llvm/pr-subscribers-lld-macho
Author: Liza Burakova (liza371)
<details>
<summary>Changes</summary>
This is part of the ld64.lld performance improvements tracked by https://github.com/llvm/llvm-project/issues/222068 and https://github.com/llvm/llvm-project/issues/238938.
Currently, command-line object files and archive member symbol tables are still parsed and scanned one file at a time as Driver::processFile() encounters them.
This change splits ObjFile::parseLazy() into scanLazy() and registerLazy(), and queues command-line object files and archive members via parseLater(). parseBatch() runs scanLazy() and parsePrepare() in parallel across the batch before running registerLazy() and parseFinish() serially in command-line order, flushing before any other input type.
Locally, this shows an ~8% link-time performance improvement over the baseline.
AI use: this change is based off a prototype developed by an LLM.
This commit is output-preserving.
---
Full diff: https://github.com/llvm/llvm-project/pull/240984.diff
3 Files Affected:
- (modified) lld/MachO/Driver.cpp (+14-5)
- (modified) lld/MachO/InputFiles.cpp (+90-28)
- (modified) lld/MachO/InputFiles.h (+19-8)
``````````diff
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f857f466fc6c6..58fc530621de2 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -416,6 +416,8 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
InputFile *newFile = nullptr;
file_magic magic = identify_magic(mbref.getBuffer());
+ if (magic != file_magic::macho_object && magic != file_magic::archive)
+ parsePendingObjects();
switch (magic) {
case file_magic::archive: {
bool isCommandLineLoad = loadType != LoadType::LCLinkerOption;
@@ -466,7 +468,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
reason = "-all_load";
break;
}
- if (Error e = file->fetch(c, reason)) {
+ if (Error e = file->fetch(c, reason, /*deferParse=*/true)) {
if (config->warnThinArchiveMissingMembers)
warn(toString(file) + ": " + reason +
" failed to load archive member: " + toString(std::move(e)));
@@ -482,7 +484,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
if (file->getArchive().hasSymbolTable()) {
for (const object::Archive::Symbol &sym : file->getArchive().symbols())
if (sym.getName().starts_with(objc::symbol_names::klass))
- file->fetch(sym);
+ file->fetch(sym, /*deferParse=*/true);
}
// TODO: no need to look for ObjC sections for a given archive member if
@@ -506,7 +508,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
archiveContents->push_back({path, isLazy, *mb});
if (!hasObjCSection(*mb))
continue;
- if (Error e = file->fetch(c, "-ObjC"))
+ if (Error e = file->fetch(c, "-ObjC", /*deferParse=*/true))
error(toString(file) + ": -ObjC failed to load archive member: " +
toString(std::move(e)));
}
@@ -521,9 +523,16 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
newFile = file;
break;
}
- case file_magic::macho_object:
- newFile = make<ObjFile>(mbref, getModTime(path), "", isLazy);
+ case file_magic::macho_object: {
+ auto *obj = make<ObjFile>(mbref, getModTime(path), "", isLazy,
+ /*forceHidden=*/false, /*compatArch=*/true,
+ /*builtFromBitcode=*/false,
+ /*deferParse=*/!isLazy);
+ if (!isLazy)
+ parseLater(*obj);
+ newFile = obj;
break;
+ }
case file_magic::macho_dynamically_linked_shared_lib:
case file_magic::macho_dynamically_linked_shared_lib_stub:
case file_magic::tapi_file:
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index c1356d06920a2..0075557e5c779 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -1110,11 +1110,13 @@ void ObjFile::parseLinkerOptions(SmallVectorImpl<StringRef> &LCLinkerOptions) {
SmallVector<StringRef> macho::unprocessedLCLinkerOptions;
ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
bool lazy, bool forceHidden, bool compatArch,
- bool builtFromBitcode)
+ bool builtFromBitcode, bool deferParse)
: InputFile(ObjKind, mb, lazy), modTime(modTime), forceHidden(forceHidden),
builtFromBitcode(builtFromBitcode) {
this->archiveName = std::string(archiveName);
this->compatArch = compatArch;
+ if (deferParse)
+ return;
if (lazy) {
if (target->wordSize == 8)
parseLazy<LP64>();
@@ -1243,6 +1245,13 @@ template <class LP> void ObjFile::parseFinish() {
}
template <class LP> void ObjFile::parseLazy() {
+ scanLazy<LP>();
+ registerLazy();
+}
+
+// The per-file half of parseLazy(): collect the names this file defines, with
+// their hashes. Touches no global state, so it can run for many files at once.
+template <class LP> void ObjFile::scanLazy() {
using Header = typename LP::mach_header;
using NList = typename LP::nlist;
@@ -1265,12 +1274,19 @@ template <class LP> void ObjFile::parseLazy() {
for (const auto &[i, sym] : llvm::enumerate(nList)) {
if ((sym.n_type & N_EXT) && !isUndef(sym)) {
// TODO: Bound checking
- StringRef name = strtab + sym.n_strx;
- symbols[i] = symtab->addLazyObject(name, *this);
+ lazyDefineds.push_back(
+ {CachedHashStringRef(strtab + sym.n_strx), static_cast<uint32_t>(i)});
}
}
}
+// The half of parseLazy() that adds to the symbol table.
+void ObjFile::registerLazy() {
+ for (const PendingSymbol &p : lazyDefineds)
+ symbols[p.symIndex] = symtab->addLazyObject(p.name, *this);
+ lazyDefineds = {};
+}
+
void ObjFile::parseDebugInfo() {
std::unique_ptr<DwarfObject> dObj = DwarfObject::create(this);
if (!dObj)
@@ -2476,12 +2492,13 @@ void ArchiveFile::addLazySymbols() {
continue;
if (!seenLazy.insert(c.getChildOffset()).second)
continue;
- auto childFile = childToObjectFile(c, /*lazy=*/true);
+ auto childFile = childToObjectFile(c, /*lazy=*/true, /*deferParse=*/true);
if (!childFile) {
llvm::consumeError(childFile.takeError());
unopenable.insert(c.getChildOffset());
continue;
}
+ parseLater(**childFile);
inputFiles.insert(*childFile);
}
if (e)
@@ -2509,17 +2526,17 @@ void ArchiveFile::addLazySymbols() {
static Expected<InputFile *>
loadArchiveMember(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
uint64_t offsetInArchive, bool forceHidden, bool compatArch,
- bool lazy) {
+ bool lazy, bool deferParse) {
if (config->zeroModTime)
modTime = 0;
switch (identify_magic(mb.getBuffer())) {
case file_magic::macho_object:
return make<ObjFile>(mb, modTime, archiveName, lazy, forceHidden,
- compatArch);
+ compatArch, /*builtFromBitcode=*/false, deferParse);
case file_magic::bitcode:
return make<BitcodeFile>(mb, archiveName, offsetInArchive, lazy,
- forceHidden, compatArch);
+ forceHidden, compatArch, deferParse);
default:
return createStringError(inconvertibleErrorCode(),
mb.getBufferIdentifier() +
@@ -2527,19 +2544,22 @@ loadArchiveMember(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
}
}
-Error ArchiveFile::fetch(const object::Archive::Child &c, StringRef reason) {
+Error ArchiveFile::fetch(const object::Archive::Child &c, StringRef reason,
+ bool deferParse) {
if (!seen.insert(c.getChildOffset()).second)
return Error::success();
- auto childFile = childToObjectFile(c, /*lazy=*/false);
+ auto childFile = childToObjectFile(c, /*lazy=*/false, deferParse);
if (!childFile)
return childFile.takeError();
+ if (deferParse)
+ parseLater(**childFile);
inputFiles.insert(*childFile);
printArchiveMemberLoad(reason, *childFile);
return Error::success();
}
-void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
+void ArchiveFile::fetch(const object::Archive::Symbol &sym, bool deferParse) {
object::Archive::Child c =
CHECK(sym.getMember(), toString(this) +
": could not get the member defining symbol " +
@@ -2552,14 +2572,14 @@ void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
// ld64 doesn't demangle sym here even with -demangle.
// Match that: intentionally don't call toMachOString().
- if (Error e = fetch(c, symCopy.getName()))
+ if (Error e = fetch(c, symCopy.getName(), deferParse))
error(toString(this) + ": could not get the member defining symbol " +
toMachOString(symCopy) + ": " + toString(std::move(e)));
}
Expected<InputFile *>
ArchiveFile::childToObjectFile(const llvm::object::Archive::Child &c,
- bool lazy) {
+ bool lazy, bool deferParse) {
Expected<MemoryBufferRef> mb = c.getMemoryBufferRef();
if (!mb)
return mb.takeError();
@@ -2569,7 +2589,8 @@ ArchiveFile::childToObjectFile(const llvm::object::Archive::Child &c,
return modTime.takeError();
return loadArchiveMember(*mb, toTimeT(*modTime), getName(),
- c.getChildOffset(), forceHidden, compatArch, lazy);
+ c.getChildOffset(), forceHidden, compatArch, lazy,
+ deferParse);
}
static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
@@ -2608,7 +2629,7 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
uint64_t offsetInArchive, bool lazy, bool forceHidden,
- bool compatArch)
+ bool compatArch, bool deferParse)
: InputFile(BitcodeKind, mb, lazy), forceHidden(forceHidden) {
this->archiveName = std::string(archiveName);
this->compatArch = compatArch;
@@ -2634,6 +2655,8 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
sys::path::filename(path) + ")" +
utostr(offsetInArchive)));
obj = check(lto::InputFile::create(mbref));
+ if (deferParse)
+ return;
if (lazy)
parseLazy();
else
@@ -2674,8 +2697,56 @@ std::string macho::replaceThinLTOSuffix(StringRef path) {
return std::string(path);
}
+static std::vector<InputFile *> pendingObjects;
+
static std::vector<ObjFile *> pendingExtracts;
+// Parses a batch of files: the per-file half for all of them at once, then the
+// half that inserts into the symbol table one file after another, so symbol
+// resolution still happens in file order and the output does not change. A
+// lazy file only registers the symbols it defines.
+static void parseBatch(ArrayRef<InputFile *> files) {
+ parallelForEach(files, [](InputFile *f) {
+ auto *obj = dyn_cast<ObjFile>(f);
+ if (!obj)
+ return;
+ if (obj->lazy) {
+ if (target->wordSize == 8)
+ obj->scanLazy<LP64>();
+ else
+ obj->scanLazy<ILP32>();
+ } else {
+ if (target->wordSize == 8)
+ obj->parsePrepare<LP64>();
+ else
+ obj->parsePrepare<ILP32>();
+ }
+ });
+ for (InputFile *f : files) {
+ if (auto *obj = dyn_cast<ObjFile>(f)) {
+ if (obj->lazy)
+ obj->registerLazy();
+ else if (target->wordSize == 8)
+ obj->parseFinish<LP64>();
+ else
+ obj->parseFinish<ILP32>();
+ } else {
+ auto *bitcode = cast<BitcodeFile>(f);
+ if (bitcode->lazy)
+ bitcode->parseLazy();
+ else
+ bitcode->parse();
+ }
+ }
+}
+
+void macho::parseLater(InputFile &file) { pendingObjects.push_back(&file); }
+
+void macho::parsePendingObjects() {
+ parseBatch(pendingObjects);
+ pendingObjects.clear();
+}
+
void macho::extract(InputFile &file, StringRef reason) {
if (!file.lazy)
return;
@@ -2696,22 +2767,13 @@ void macho::extract(InputFile &file, StringRef reason) {
}
void macho::parsePendingExtracts() {
+ parsePendingObjects();
+
for (size_t done = 0; done < pendingExtracts.size();) {
size_t end = pendingExtracts.size();
- std::vector<ObjFile *> round(pendingExtracts.begin() + done,
- pendingExtracts.begin() + end);
- parallelForEach(round, [](ObjFile *f) {
- if (target->wordSize == 8)
- f->parsePrepare<LP64>();
- else
- f->parsePrepare<ILP32>();
- });
- for (ObjFile *f : round) {
- if (target->wordSize == 8)
- f->parseFinish<LP64>();
- else
- f->parseFinish<ILP32>();
- }
+ std::vector<InputFile *> round(pendingExtracts.begin() + done,
+ pendingExtracts.begin() + end);
+ parseBatch(round);
done = end;
}
pendingExtracts.clear();
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
index 2156ada339750..cd935daf68778 100644
--- a/lld/MachO/InputFiles.h
+++ b/lld/MachO/InputFiles.h
@@ -161,7 +161,7 @@ class ObjFile final : public InputFile {
public:
ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
bool lazy = false, bool forceHidden = false, bool compatArch = true,
- bool builtFromBitcode = false);
+ bool builtFromBitcode = false, bool deferParse = false);
ArrayRef<llvm::MachO::data_in_code_entry> getDataInCode() const;
ArrayRef<uint8_t> getOptimizationHints() const;
template <class LP> void parse();
@@ -170,6 +170,10 @@ class ObjFile final : public InputFile {
// the half that inserts into the symbol table, and has to run in file order.
template <class LP> void parsePrepare();
template <class LP> void parseFinish();
+ // parseLazy() split the same way: scanLazy() collects the names this file
+ // defines, registerLazy() adds them to the symbol table.
+ template <class LP> void scanLazy();
+ void registerLazy();
template <class LP>
void parseLinkerOptions(llvm::SmallVectorImpl<StringRef> &LinkerOptions);
void parseDeferredRelocations();
@@ -214,6 +218,7 @@ class ObjFile final : public InputFile {
};
std::vector<PendingSymbol> undefineds;
std::vector<PendingSymbol> nonSectionSymbols;
+ std::vector<PendingSymbol> lazyDefineds;
// An external section symbol, and the subsection it was found to be in.
struct PendingDefined {
llvm::CachedHashStringRef name;
@@ -320,16 +325,17 @@ class ArchiveFile final : public InputFile {
explicit ArchiveFile(std::unique_ptr<llvm::object::Archive> &&file,
bool forceHidden);
void addLazySymbols();
- void fetch(const llvm::object::Archive::Symbol &);
+ void fetch(const llvm::object::Archive::Symbol &, bool deferParse = false);
// LLD normally doesn't use Error for error-handling, but the underlying
// Archive library does, so this is the cleanest way to wrap it.
- Error fetch(const llvm::object::Archive::Child &, StringRef reason);
+ Error fetch(const llvm::object::Archive::Child &, StringRef reason,
+ bool deferParse = false);
const llvm::object::Archive &getArchive() const { return *file; };
static bool classof(const InputFile *f) { return f->kind() == ArchiveKind; }
private:
Expected<InputFile *> childToObjectFile(const llvm::object::Archive::Child &c,
- bool lazy);
+ bool lazy, bool deferParse = false);
std::unique_ptr<llvm::object::Archive> file;
// Keep track of children fetched from the archive by tracking
// which address offsets have been fetched already.
@@ -343,15 +349,14 @@ class BitcodeFile final : public InputFile {
public:
explicit BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
uint64_t offsetInArchive, bool lazy = false,
- bool forceHidden = false, bool compatArch = true);
+ bool forceHidden = false, bool compatArch = true,
+ bool deferParse = false);
static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
void parse();
+ void parseLazy();
std::unique_ptr<llvm::lto::InputFile> obj;
bool forceHidden;
-
-private:
- void parseLazy();
};
extern llvm::SetVector<InputFile *> inputFiles;
@@ -366,6 +371,12 @@ void extract(InputFile &file, StringRef reason);
// Parses the files extract() has pulled into the link. May pull in more.
void parsePendingExtracts();
+// Queues an object file from the command line to be parsed by the next
+// parsePendingObjects(), together with the other files queued since the last
+// one. The driver calls that before anything else that adds to the symbol
+// table, so files still resolve in command-line order.
+void parseLater(InputFile &file);
+void parsePendingObjects();
namespace detail {
``````````
</details>
https://github.com/llvm/llvm-project/pull/240984
More information about the llvm-branch-commits
mailing list