[llvm-branch-commits] [lld] [lld][MachO] Batch command-line object files and archive member scanning in parallel (PR #240984)

Liza Burakova via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Tue Nov 10 12:04:38 PST 2026


https://github.com/liza371 created https://github.com/llvm/llvm-project/pull/240984

This is part of the ld64.lld performance improvements tracked by https://github.com/llvm/llvm-project/issues/222068 and https://github.com/llvm/llvm-project/issues/238938.

Currently, command-line object files and archive member symbol tables are still parsed and scanned one file at a time as Driver::processFile() encounters them.

This change splits ObjFile::parseLazy() into scanLazy() and registerLazy(), and queues command-line object files and archive members via parseLater(). parseBatch() runs scanLazy() and parsePrepare() in parallel across the batch before running registerLazy() and parseFinish() serially in command-line order, flushing before any other input type.

Locally, this shows an ~8% link-time performance improvement over the baseline.

AI use: this change is based off a prototype developed by an LLM.

This commit is output-preserving.

>From 29a192248b293ba0005e6ac8a3c0897126eff2dd Mon Sep 17 00:00:00 2001
From: Liza Burakova <liza at chromium.org>
Date: Mon, 9 Nov 2026 16:56:50 -0500
Subject: [PATCH] [lld][MachO] Batch command-line object files and archive
 member scanning in parallel

This is part of the ld64.lld performance improvements tracked by

Currently, command-line object files and archive member symbol tables
are still parsed and scanned one file at a time as Driver::processFile()
encounters them.

This change splits ObjFile::parseLazy() into scanLazy() and
registerLazy(), and queues command-line object files and archive members
via parseLater(). parseBatch() runs scanLazy() and parsePrepare() in
parallel across the batch before running registerLazy() and
parseFinish() serially in command-line order, flushing before any other
input type.

Locally, this shows an ~8% link-time performance improvement over the
baseline.

AI use: this change is based off a prototype developed by an LLM.

This commit is output-preserving.
---
 lld/MachO/Driver.cpp     |  19 +++++--
 lld/MachO/InputFiles.cpp | 118 +++++++++++++++++++++++++++++----------
 lld/MachO/InputFiles.h   |  27 ++++++---
 3 files changed, 123 insertions(+), 41 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f857f466fc6c6..58fc530621de2 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -416,6 +416,8 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
   InputFile *newFile = nullptr;
 
   file_magic magic = identify_magic(mbref.getBuffer());
+  if (magic != file_magic::macho_object && magic != file_magic::archive)
+    parsePendingObjects();
   switch (magic) {
   case file_magic::archive: {
     bool isCommandLineLoad = loadType != LoadType::LCLinkerOption;
@@ -466,7 +468,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
             reason = "-all_load";
             break;
           }
-          if (Error e = file->fetch(c, reason)) {
+          if (Error e = file->fetch(c, reason, /*deferParse=*/true)) {
             if (config->warnThinArchiveMissingMembers)
               warn(toString(file) + ": " + reason +
                    " failed to load archive member: " + toString(std::move(e)));
@@ -482,7 +484,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
       if (file->getArchive().hasSymbolTable()) {
         for (const object::Archive::Symbol &sym : file->getArchive().symbols())
           if (sym.getName().starts_with(objc::symbol_names::klass))
-            file->fetch(sym);
+            file->fetch(sym, /*deferParse=*/true);
       }
 
       // TODO: no need to look for ObjC sections for a given archive member if
@@ -506,7 +508,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
             archiveContents->push_back({path, isLazy, *mb});
           if (!hasObjCSection(*mb))
             continue;
-          if (Error e = file->fetch(c, "-ObjC"))
+          if (Error e = file->fetch(c, "-ObjC", /*deferParse=*/true))
             error(toString(file) + ": -ObjC failed to load archive member: " +
                   toString(std::move(e)));
         }
@@ -521,9 +523,16 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
     newFile = file;
     break;
   }
-  case file_magic::macho_object:
-    newFile = make<ObjFile>(mbref, getModTime(path), "", isLazy);
+  case file_magic::macho_object: {
+    auto *obj = make<ObjFile>(mbref, getModTime(path), "", isLazy,
+                              /*forceHidden=*/false, /*compatArch=*/true,
+                              /*builtFromBitcode=*/false,
+                              /*deferParse=*/!isLazy);
+    if (!isLazy)
+      parseLater(*obj);
+    newFile = obj;
     break;
+  }
   case file_magic::macho_dynamically_linked_shared_lib:
   case file_magic::macho_dynamically_linked_shared_lib_stub:
   case file_magic::tapi_file:
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index c1356d06920a2..0075557e5c779 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -1110,11 +1110,13 @@ void ObjFile::parseLinkerOptions(SmallVectorImpl<StringRef> &LCLinkerOptions) {
 SmallVector<StringRef> macho::unprocessedLCLinkerOptions;
 ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
                  bool lazy, bool forceHidden, bool compatArch,
-                 bool builtFromBitcode)
+                 bool builtFromBitcode, bool deferParse)
     : InputFile(ObjKind, mb, lazy), modTime(modTime), forceHidden(forceHidden),
       builtFromBitcode(builtFromBitcode) {
   this->archiveName = std::string(archiveName);
   this->compatArch = compatArch;
+  if (deferParse)
+    return;
   if (lazy) {
     if (target->wordSize == 8)
       parseLazy<LP64>();
@@ -1243,6 +1245,13 @@ template <class LP> void ObjFile::parseFinish() {
 }
 
 template <class LP> void ObjFile::parseLazy() {
+  scanLazy<LP>();
+  registerLazy();
+}
+
+// The per-file half of parseLazy(): collect the names this file defines, with
+// their hashes. Touches no global state, so it can run for many files at once.
+template <class LP> void ObjFile::scanLazy() {
   using Header = typename LP::mach_header;
   using NList = typename LP::nlist;
 
@@ -1265,12 +1274,19 @@ template <class LP> void ObjFile::parseLazy() {
   for (const auto &[i, sym] : llvm::enumerate(nList)) {
     if ((sym.n_type & N_EXT) && !isUndef(sym)) {
       // TODO: Bound checking
-      StringRef name = strtab + sym.n_strx;
-      symbols[i] = symtab->addLazyObject(name, *this);
+      lazyDefineds.push_back(
+          {CachedHashStringRef(strtab + sym.n_strx), static_cast<uint32_t>(i)});
     }
   }
 }
 
+// The half of parseLazy() that adds to the symbol table.
+void ObjFile::registerLazy() {
+  for (const PendingSymbol &p : lazyDefineds)
+    symbols[p.symIndex] = symtab->addLazyObject(p.name, *this);
+  lazyDefineds = {};
+}
+
 void ObjFile::parseDebugInfo() {
   std::unique_ptr<DwarfObject> dObj = DwarfObject::create(this);
   if (!dObj)
@@ -2476,12 +2492,13 @@ void ArchiveFile::addLazySymbols() {
       continue;
     if (!seenLazy.insert(c.getChildOffset()).second)
       continue;
-    auto childFile = childToObjectFile(c, /*lazy=*/true);
+    auto childFile = childToObjectFile(c, /*lazy=*/true, /*deferParse=*/true);
     if (!childFile) {
       llvm::consumeError(childFile.takeError());
       unopenable.insert(c.getChildOffset());
       continue;
     }
+    parseLater(**childFile);
     inputFiles.insert(*childFile);
   }
   if (e)
@@ -2509,17 +2526,17 @@ void ArchiveFile::addLazySymbols() {
 static Expected<InputFile *>
 loadArchiveMember(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
                   uint64_t offsetInArchive, bool forceHidden, bool compatArch,
-                  bool lazy) {
+                  bool lazy, bool deferParse) {
   if (config->zeroModTime)
     modTime = 0;
 
   switch (identify_magic(mb.getBuffer())) {
   case file_magic::macho_object:
     return make<ObjFile>(mb, modTime, archiveName, lazy, forceHidden,
-                         compatArch);
+                         compatArch, /*builtFromBitcode=*/false, deferParse);
   case file_magic::bitcode:
     return make<BitcodeFile>(mb, archiveName, offsetInArchive, lazy,
-                             forceHidden, compatArch);
+                             forceHidden, compatArch, deferParse);
   default:
     return createStringError(inconvertibleErrorCode(),
                              mb.getBufferIdentifier() +
@@ -2527,19 +2544,22 @@ loadArchiveMember(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
   }
 }
 
-Error ArchiveFile::fetch(const object::Archive::Child &c, StringRef reason) {
+Error ArchiveFile::fetch(const object::Archive::Child &c, StringRef reason,
+                         bool deferParse) {
   if (!seen.insert(c.getChildOffset()).second)
     return Error::success();
-  auto childFile = childToObjectFile(c, /*lazy=*/false);
+  auto childFile = childToObjectFile(c, /*lazy=*/false, deferParse);
   if (!childFile)
     return childFile.takeError();
 
+  if (deferParse)
+    parseLater(**childFile);
   inputFiles.insert(*childFile);
   printArchiveMemberLoad(reason, *childFile);
   return Error::success();
 }
 
-void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
+void ArchiveFile::fetch(const object::Archive::Symbol &sym, bool deferParse) {
   object::Archive::Child c =
       CHECK(sym.getMember(), toString(this) +
                                  ": could not get the member defining symbol " +
@@ -2552,14 +2572,14 @@ void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
 
   // ld64 doesn't demangle sym here even with -demangle.
   // Match that: intentionally don't call toMachOString().
-  if (Error e = fetch(c, symCopy.getName()))
+  if (Error e = fetch(c, symCopy.getName(), deferParse))
     error(toString(this) + ": could not get the member defining symbol " +
           toMachOString(symCopy) + ": " + toString(std::move(e)));
 }
 
 Expected<InputFile *>
 ArchiveFile::childToObjectFile(const llvm::object::Archive::Child &c,
-                               bool lazy) {
+                               bool lazy, bool deferParse) {
   Expected<MemoryBufferRef> mb = c.getMemoryBufferRef();
   if (!mb)
     return mb.takeError();
@@ -2569,7 +2589,8 @@ ArchiveFile::childToObjectFile(const llvm::object::Archive::Child &c,
     return modTime.takeError();
 
   return loadArchiveMember(*mb, toTimeT(*modTime), getName(),
-                           c.getChildOffset(), forceHidden, compatArch, lazy);
+                           c.getChildOffset(), forceHidden, compatArch, lazy,
+                           deferParse);
 }
 
 static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
@@ -2608,7 +2629,7 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
 
 BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
                          uint64_t offsetInArchive, bool lazy, bool forceHidden,
-                         bool compatArch)
+                         bool compatArch, bool deferParse)
     : InputFile(BitcodeKind, mb, lazy), forceHidden(forceHidden) {
   this->archiveName = std::string(archiveName);
   this->compatArch = compatArch;
@@ -2634,6 +2655,8 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
                                                sys::path::filename(path) + ")" +
                                                utostr(offsetInArchive)));
   obj = check(lto::InputFile::create(mbref));
+  if (deferParse)
+    return;
   if (lazy)
     parseLazy();
   else
@@ -2674,8 +2697,56 @@ std::string macho::replaceThinLTOSuffix(StringRef path) {
   return std::string(path);
 }
 
+static std::vector<InputFile *> pendingObjects;
+
 static std::vector<ObjFile *> pendingExtracts;
 
+// Parses a batch of files: the per-file half for all of them at once, then the
+// half that inserts into the symbol table one file after another, so symbol
+// resolution still happens in file order and the output does not change. A
+// lazy file only registers the symbols it defines.
+static void parseBatch(ArrayRef<InputFile *> files) {
+  parallelForEach(files, [](InputFile *f) {
+    auto *obj = dyn_cast<ObjFile>(f);
+    if (!obj)
+      return;
+    if (obj->lazy) {
+      if (target->wordSize == 8)
+        obj->scanLazy<LP64>();
+      else
+        obj->scanLazy<ILP32>();
+    } else {
+      if (target->wordSize == 8)
+        obj->parsePrepare<LP64>();
+      else
+        obj->parsePrepare<ILP32>();
+    }
+  });
+  for (InputFile *f : files) {
+    if (auto *obj = dyn_cast<ObjFile>(f)) {
+      if (obj->lazy)
+        obj->registerLazy();
+      else if (target->wordSize == 8)
+        obj->parseFinish<LP64>();
+      else
+        obj->parseFinish<ILP32>();
+    } else {
+      auto *bitcode = cast<BitcodeFile>(f);
+      if (bitcode->lazy)
+        bitcode->parseLazy();
+      else
+        bitcode->parse();
+    }
+  }
+}
+
+void macho::parseLater(InputFile &file) { pendingObjects.push_back(&file); }
+
+void macho::parsePendingObjects() {
+  parseBatch(pendingObjects);
+  pendingObjects.clear();
+}
+
 void macho::extract(InputFile &file, StringRef reason) {
   if (!file.lazy)
     return;
@@ -2696,22 +2767,13 @@ void macho::extract(InputFile &file, StringRef reason) {
 }
 
 void macho::parsePendingExtracts() {
+  parsePendingObjects();
+
   for (size_t done = 0; done < pendingExtracts.size();) {
     size_t end = pendingExtracts.size();
-    std::vector<ObjFile *> round(pendingExtracts.begin() + done,
-                                 pendingExtracts.begin() + end);
-    parallelForEach(round, [](ObjFile *f) {
-      if (target->wordSize == 8)
-        f->parsePrepare<LP64>();
-      else
-        f->parsePrepare<ILP32>();
-    });
-    for (ObjFile *f : round) {
-      if (target->wordSize == 8)
-        f->parseFinish<LP64>();
-      else
-        f->parseFinish<ILP32>();
-    }
+    std::vector<InputFile *> round(pendingExtracts.begin() + done,
+                                   pendingExtracts.begin() + end);
+    parseBatch(round);
     done = end;
   }
   pendingExtracts.clear();
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
index 2156ada339750..cd935daf68778 100644
--- a/lld/MachO/InputFiles.h
+++ b/lld/MachO/InputFiles.h
@@ -161,7 +161,7 @@ class ObjFile final : public InputFile {
 public:
   ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName,
           bool lazy = false, bool forceHidden = false, bool compatArch = true,
-          bool builtFromBitcode = false);
+          bool builtFromBitcode = false, bool deferParse = false);
   ArrayRef<llvm::MachO::data_in_code_entry> getDataInCode() const;
   ArrayRef<uint8_t> getOptimizationHints() const;
   template <class LP> void parse();
@@ -170,6 +170,10 @@ class ObjFile final : public InputFile {
   // the half that inserts into the symbol table, and has to run in file order.
   template <class LP> void parsePrepare();
   template <class LP> void parseFinish();
+  // parseLazy() split the same way: scanLazy() collects the names this file
+  // defines, registerLazy() adds them to the symbol table.
+  template <class LP> void scanLazy();
+  void registerLazy();
   template <class LP>
   void parseLinkerOptions(llvm::SmallVectorImpl<StringRef> &LinkerOptions);
   void parseDeferredRelocations();
@@ -214,6 +218,7 @@ class ObjFile final : public InputFile {
   };
   std::vector<PendingSymbol> undefineds;
   std::vector<PendingSymbol> nonSectionSymbols;
+  std::vector<PendingSymbol> lazyDefineds;
   // An external section symbol, and the subsection it was found to be in.
   struct PendingDefined {
     llvm::CachedHashStringRef name;
@@ -320,16 +325,17 @@ class ArchiveFile final : public InputFile {
   explicit ArchiveFile(std::unique_ptr<llvm::object::Archive> &&file,
                        bool forceHidden);
   void addLazySymbols();
-  void fetch(const llvm::object::Archive::Symbol &);
+  void fetch(const llvm::object::Archive::Symbol &, bool deferParse = false);
   // LLD normally doesn't use Error for error-handling, but the underlying
   // Archive library does, so this is the cleanest way to wrap it.
-  Error fetch(const llvm::object::Archive::Child &, StringRef reason);
+  Error fetch(const llvm::object::Archive::Child &, StringRef reason,
+              bool deferParse = false);
   const llvm::object::Archive &getArchive() const { return *file; };
   static bool classof(const InputFile *f) { return f->kind() == ArchiveKind; }
 
 private:
   Expected<InputFile *> childToObjectFile(const llvm::object::Archive::Child &c,
-                                          bool lazy);
+                                          bool lazy, bool deferParse = false);
   std::unique_ptr<llvm::object::Archive> file;
   // Keep track of children fetched from the archive by tracking
   // which address offsets have been fetched already.
@@ -343,15 +349,14 @@ class BitcodeFile final : public InputFile {
 public:
   explicit BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
                        uint64_t offsetInArchive, bool lazy = false,
-                       bool forceHidden = false, bool compatArch = true);
+                       bool forceHidden = false, bool compatArch = true,
+                       bool deferParse = false);
   static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
   void parse();
+  void parseLazy();
 
   std::unique_ptr<llvm::lto::InputFile> obj;
   bool forceHidden;
-
-private:
-  void parseLazy();
 };
 
 extern llvm::SetVector<InputFile *> inputFiles;
@@ -366,6 +371,12 @@ void extract(InputFile &file, StringRef reason);
 
 // Parses the files extract() has pulled into the link. May pull in more.
 void parsePendingExtracts();
+// Queues an object file from the command line to be parsed by the next
+// parsePendingObjects(), together with the other files queued since the last
+// one. The driver calls that before anything else that adds to the symbol
+// table, so files still resolve in command-line order.
+void parseLater(InputFile &file);
+void parsePendingObjects();
 
 namespace detail {
 



More information about the llvm-branch-commits mailing list