[llvm-branch-commits] [lld] [lld-macho] Mark live sections and symbols in parallel (PR #240243)

Jez Ng via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 8 23:19:04 PST 2026


https://github.com/int3 updated https://github.com/llvm/llvm-project/pull/240243

>From e094f27a8baf81e96bd31698870ac52bfd422f4c Mon Sep 17 00:00:00 2001
From: Jez Ng <me at jezng.com>
Date: Sun, 8 Nov 2026 01:12:22 -0800
Subject: [PATCH] [lld-macho] Mark live sections and symbols in parallel

---
 lld/MachO/InputSection.cpp |  13 ++-
 lld/MachO/InputSection.h   |  24 ++++-
 lld/MachO/MarkLive.cpp     | 181 +++++++++++++++++++++++++------------
 lld/MachO/Options.td       |   3 +-
 lld/MachO/Symbols.h        |   2 +-
 lld/test/MachO/why-live.s  |  24 ++---
 6 files changed, 164 insertions(+), 83 deletions(-)

diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp
index ebbe5ca5006cf0..1d8bbdc1f88fef 100644
--- a/lld/MachO/InputSection.cpp
+++ b/lld/MachO/InputSection.cpp
@@ -328,8 +328,8 @@ StringPiece &CStringInputSection::getStringPiece(uint64_t off) {
   if (off >= data.size())
     fatal(toString(this) + ": offset is outside the section");
 
-  auto it =
-      partition_point(pieces, [=](StringPiece p) { return p.inSecOff <= off; });
+  auto it = partition_point(
+      pieces, [=](const StringPiece &p) { return p.inSecOff <= off; });
   return it[-1];
 }
 
@@ -341,8 +341,8 @@ size_t CStringInputSection::getStringPieceIndex(uint64_t off) const {
   if (off >= data.size())
     fatal(toString(this) + ": offset is outside the section");
 
-  auto it =
-      partition_point(pieces, [=](StringPiece p) { return p.inSecOff <= off; });
+  auto it = partition_point(
+      pieces, [=](const StringPiece &p) { return p.inSecOff <= off; });
   return std::distance(pieces.begin(), it) - 1;
 }
 
@@ -370,7 +370,10 @@ WordLiteralInputSection::WordLiteralInputSection(const Section &section,
     llvm_unreachable("invalid literal section type");
   }
 
-  live.resize(data.size() >> power2LiteralSize, !config->deadStrip);
+  size_t numLiterals = data.size() >> power2LiteralSize;
+  live = std::make_unique<std::atomic<bool>[]>(numLiterals);
+  if (!config->deadStrip)
+    std::fill_n(live.get(), numLiterals, true);
 }
 
 uint64_t WordLiteralInputSection::getOffset(uint64_t off) const {
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 5a58bdc099ede4..349bf1393a3121 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -16,10 +16,13 @@
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Memory.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/SwapByteOrder.h"
+
+#include <atomic>
+#include <memory>
 
 namespace lld {
 namespace macho {
@@ -190,6 +193,17 @@ struct StringPiece {
 
   StringPiece(uint64_t off, uint32_t hash)
       : inSecOff(off), live(!config->deadStrip), hash(hash) {}
+
+  // Marks the piece live; safe to call from several threads at once. No atomic
+  // OR is needed: while markLive runs, `hash` does not change and `live` only
+  // goes from false to true, so threads racing here all store the same value.
+  void markLive() {
+    auto *word = reinterpret_cast<std::atomic<uint32_t> *>(&inSecOff + 1);
+    constexpr uint32_t liveBit = llvm::sys::IsBigEndianHost ? (1U << 31) : 1U;
+    uint32_t w = word->load(std::memory_order_relaxed);
+    if (!(w & liveBit))
+      word->store(w | liveBit, std::memory_order_relaxed);
+  }
 };
 
 static_assert(sizeof(StringPiece) == 16, "StringPiece is too big!");
@@ -213,7 +227,7 @@ class CStringInputSection final : public InputSection {
 
   uint64_t getOffset(uint64_t off) const override;
   bool isLive(uint64_t off) const override { return getStringPiece(off).live; }
-  void markLive(uint64_t off) override { getStringPiece(off).live = true; }
+  void markLive(uint64_t off) override { getStringPiece(off).markLive(); }
   // Find the StringPiece that contains this offset.
   StringPiece &getStringPiece(uint64_t off);
   const StringPiece &getStringPiece(uint64_t off) const;
@@ -259,10 +273,10 @@ class WordLiteralInputSection final : public InputSection {
                           uint32_t align);
   uint64_t getOffset(uint64_t off) const override;
   bool isLive(uint64_t off) const override {
-    return live[off >> power2LiteralSize];
+    return live[off >> power2LiteralSize].load(std::memory_order_relaxed);
   }
   void markLive(uint64_t off) override {
-    live[off >> power2LiteralSize] = true;
+    live[off >> power2LiteralSize].store(true, std::memory_order_relaxed);
   }
 
   static bool classof(const InputSection *isec) {
@@ -272,7 +286,7 @@ class WordLiteralInputSection final : public InputSection {
 private:
   unsigned power2LiteralSize;
   // The liveness of data[off] is tracked by live[off >> power2LiteralSize].
-  llvm::BitVector live;
+  std::unique_ptr<std::atomic<bool>[]> live;
 };
 
 inline uint8_t sectionType(uint32_t flags) {
diff --git a/lld/MachO/MarkLive.cpp b/lld/MachO/MarkLive.cpp
index cf348cb22bbf4a..924ba94ad3aef3 100644
--- a/lld/MachO/MarkLive.cpp
+++ b/lld/MachO/MarkLive.cpp
@@ -15,8 +15,11 @@
 #include "UnwindInfoSection.h"
 
 #include "lld/Common/ErrorHandler.h"
+#include "llvm/Support/Parallel.h"
 #include "llvm/Support/TimeProfiler.h"
 
+#include <mutex>
+
 namespace lld::macho {
 
 using namespace llvm;
@@ -41,6 +44,18 @@ class MarkLive {
   virtual ~MarkLive() = default;
 };
 
+// Sets `flag` and returns true if no other thread set it first. C++17 has no
+// std::atomic_ref, so the plain bool is accessed as an atomic.
+static bool tryClaim(bool &flag) {
+  auto &atomicFlag = reinterpret_cast<std::atomic<bool> &>(flag);
+  // Most claims find the flag already set, and the load lets those skip the
+  // more expensive atomic exchange.
+  return !atomicFlag.load(std::memory_order_relaxed) &&
+         !atomicFlag.exchange(true, std::memory_order_relaxed);
+}
+
+// Marks liveness on as many threads as are available. -why_live output is
+// deterministic only with --threads=1.
 template <bool RecordWhyLive> class MarkLiveImpl : public MarkLive {
 public:
   // -why_live is a rarely used option, so we don't want support for that flag
@@ -49,41 +64,49 @@ template <bool RecordWhyLive> class MarkLiveImpl : public MarkLive {
   // from needless allocations and pointer indirections.
   using WorklistEntry =
       std::conditional_t<RecordWhyLive, WhyLiveEntry, InputSection>;
+  using Worklist = SmallVector<WorklistEntry *, 0>;
 
   void enqueue(InputSection *isec, uint64_t off) override {
-    enqueue(isec, off, nullptr);
+    enqueue(isec, off, nullptr, nextLevel);
   }
-  void addSym(Symbol *s) override { addSym(s, nullptr); }
+  void addSym(Symbol *s) override { addSym(s, nullptr, nextLevel); }
   void markTransitively() override;
 
 private:
-  void enqueue(InputSection *isec, uint64_t off, const WorklistEntry *prev);
-  // Most calls find `s` already used, so keep that check out from behind a
+  // Several threads call these at once, each passing its own worklist to
+  // receive the sections it claims.
+  void enqueue(InputSection *isec, uint64_t off, const WorklistEntry *prev,
+               Worklist &worklist) const;
+  // Most calls find `s` already claimed, so keep that check out from behind a
   // call.
-  LLVM_ATTRIBUTE_ALWAYS_INLINE void addSym(Symbol *s,
-                                           const WorklistEntry *prev);
-  void visitSymbol(Symbol *s, const WorklistEntry *prev);
+  LLVM_ATTRIBUTE_ALWAYS_INLINE void addSym(Symbol *s, const WorklistEntry *prev,
+                                           Worklist &worklist) const;
+  void visitSymbol(Symbol *s, const WorklistEntry *prev,
+                   Worklist &worklist) const;
+  void visit(WorklistEntry *entry, Worklist &worklist) const;
+  void drain();
   const InputSection *getInputSection(const WorklistEntry *) const;
   WorklistEntry *makeEntry(InputSection *, const WorklistEntry *prev) const;
 
-  // We build up a worklist of sections which have been marked as live. We
-  // only push into the worklist when we discover an unmarked section, and we
-  // mark as we push, so sections never appear twice in the list. Literal
-  // sections cannot contain references to other sections, so we only store
-  // ConcatInputSections in our worklist.
-  SmallVector<WorklistEntry *, 256> worklist;
+  // Sections that have been marked live and that the next level of drain() will
+  // visit. Literal sections cannot contain references to other sections, so we
+  // only store ConcatInputSections here.
+  Worklist nextLevel;
 };
 
+// Marks `isec` (at `off`) live. Only the thread whose claim succeeds queues a
+// section.
 template <bool RecordWhyLive>
-void MarkLiveImpl<RecordWhyLive>::enqueue(
-    InputSection *isec, uint64_t off,
-    const typename MarkLiveImpl<RecordWhyLive>::WorklistEntry *prev) {
-  if (isec->isLive(off))
-    return;
-  isec->markLive(off);
-  if (auto s = dyn_cast<ConcatInputSection>(isec)) {
+void MarkLiveImpl<RecordWhyLive>::enqueue(InputSection *isec, uint64_t off,
+                                          const WorklistEntry *prev,
+                                          Worklist &worklist) const {
+  if (auto *s = dyn_cast<ConcatInputSection>(isec)) {
+    if (!tryClaim(s->live))
+      return;
     assert(!s->isCoalescedWeak());
     worklist.push_back(makeEntry(s, prev));
+  } else {
+    isec->markLive(off);
   }
 }
 
@@ -103,32 +126,90 @@ static void printWhyLive(const Symbol *s, const WhyLiveEntry *prev) {
 }
 
 template <bool RecordWhyLive>
-void MarkLiveImpl<RecordWhyLive>::addSym(
-    Symbol *s,
-    const typename MarkLiveImpl<RecordWhyLive>::WorklistEntry *prev) {
-  if (s->used)
-    return;
-  s->used = true;
-  visitSymbol(s, prev);
+void MarkLiveImpl<RecordWhyLive>::addSym(Symbol *s, const WorklistEntry *prev,
+                                         Worklist &worklist) const {
+  if (tryClaim(s->used))
+    visitSymbol(s, prev, worklist);
 }
 
-// Marks what `s` keeps alive. This is separate from addSym() because the
-// class-stub edge makes it recursive, and GCC won't always-inline a function
-// that calls itself.
+// Marks what `s` keeps alive. Called by the thread that claimed `s`. This is
+// separate from addSym() because the class-stub edge makes it recursive, and
+// GCC won't always-inline a function that calls itself.
 template <bool RecordWhyLive>
-void MarkLiveImpl<RecordWhyLive>::visitSymbol(
-    Symbol *s,
-    const typename MarkLiveImpl<RecordWhyLive>::WorklistEntry *prev) {
+void MarkLiveImpl<RecordWhyLive>::visitSymbol(Symbol *s,
+                                              const WorklistEntry *prev,
+                                              Worklist &worklist) const {
   if constexpr (RecordWhyLive)
     if (!config->whyLive.empty() && config->whyLive.match(s->getName()))
       printWhyLive(s, prev);
   if (auto *classSym = in.objcStubs->lookupClassSymbol(s))
-    addSym(classSym, prev);
+    addSym(classSym, prev, worklist);
   if (auto *d = dyn_cast<Defined>(s)) {
     if (d->isec())
-      enqueue(d->isec(), d->value, prev);
+      enqueue(d->isec(), d->value, prev, worklist);
     if (d->unwindEntry())
-      enqueue(d->unwindEntry(), 0, prev);
+      enqueue(d->unwindEntry(), 0, prev, worklist);
+  }
+}
+
+template <bool RecordWhyLive>
+void MarkLiveImpl<RecordWhyLive>::visit(WorklistEntry *entry,
+                                        Worklist &worklist) const {
+  // Entries that get placed onto the worklist always contain
+  // ConcatInputSections. `WhyLiveEntry::prev` may point to entries that
+  // contain other types of InputSections (due to S_ATTR_LIVE_SUPPORT), but
+  // those entries should never be pushed onto the worklist.
+  auto *isec = cast<ConcatInputSection>(getInputSection(entry));
+  assert(isec->live && "We mark as live when pushing onto the worklist!");
+
+  // Mark all symbols listed in the relocation table for this section.
+  for (const Relocation &r : isec->relocs) {
+    if (auto *s = dyn_cast<Symbol *>(r.referent))
+      addSym(s, entry, worklist);
+    else
+      enqueue(cast<InputSection *>(r.referent), r.getAddend(), entry, worklist);
+  }
+  for (Defined *d : isec->symbols)
+    addSym(d, entry, worklist);
+}
+
+// Visits the sections in nextLevel, one level at a time. Workers take chunks of
+// a level, small enough that every worker gets some, and visit them depth-first
+// to take advantage of memory locality. Once one worker has run out of chunks,
+// the others finish their current visit and move the rest of their stacks to
+// nextLevel, so that the level ends and all workers share the rest in the next
+// one. With one thread, this reduces to a single depth-first walk. The last
+// join makes every live bit visible to the live-support scan, which reads them
+// with plain loads.
+template <bool RecordWhyLive> void MarkLiveImpl<RecordWhyLive>::drain() {
+  const size_t numThreads = parallel::getThreadCount();
+  while (!nextLevel.empty()) {
+    Worklist level = std::move(nextLevel);
+    nextLevel.clear();
+    std::mutex nextLevelMutex;
+    std::atomic<bool> anyWorkerIdle{false};
+    size_t chunkSize =
+        std::clamp<size_t>(level.size() / (4 * numThreads), 1, 64);
+    // Each worker starts with the chunk at its own index and then claims the
+    // chunks after the first numThreads.
+    std::atomic<size_t> next{numThreads * chunkSize};
+    parallelFor(0, numThreads, [&](size_t shard) {
+      Worklist stack;
+      for (size_t begin = shard * chunkSize; begin < level.size();
+           begin = next.fetch_add(chunkSize, std::memory_order_relaxed)) {
+        size_t end = std::min(begin + chunkSize, level.size());
+        stack.append(level.begin() + begin, level.begin() + end);
+        while (!stack.empty()) {
+          visit(stack.pop_back_val(), stack);
+          if (anyWorkerIdle.load(std::memory_order_relaxed)) {
+            std::lock_guard<std::mutex> lock(nextLevelMutex);
+            nextLevel.append(stack.begin(), stack.end());
+            stack.clear();
+          }
+        }
+      }
+      anyWorkerIdle.store(true, std::memory_order_relaxed);
+    });
   }
 }
 
@@ -151,7 +232,7 @@ MarkLiveImpl<RecordWhyLive>::makeEntry(
       assert(!prev);
       return nullptr;
     }
-    return make<WhyLiveEntry>(isec, prev);
+    return makeThreadLocal<WhyLiveEntry>(isec, prev);
   } else {
     return isec;
   }
@@ -161,25 +242,7 @@ template <bool RecordWhyLive>
 void MarkLiveImpl<RecordWhyLive>::markTransitively() {
   do {
     // Mark things reachable from GC roots as live.
-    while (!worklist.empty()) {
-      WorklistEntry *entry = worklist.pop_back_val();
-      // Entries that get placed onto the worklist always contain
-      // ConcatInputSections. `WhyLiveEntry::prev` may point to entries that
-      // contain other types of InputSections (due to S_ATTR_LIVE_SUPPORT), but
-      // those entries should never be pushed onto the worklist.
-      auto *isec = cast<ConcatInputSection>(getInputSection(entry));
-      assert(isec->live && "We mark as live when pushing onto the worklist!");
-
-      // Mark all symbols listed in the relocation table for this section.
-      for (const Relocation &r : isec->relocs) {
-        if (auto *s = dyn_cast<Symbol *>(r.referent))
-          addSym(s, entry);
-        else
-          enqueue(cast<InputSection *>(r.referent), r.getAddend(), entry);
-      }
-      for (Defined *d : getInputSection(entry)->symbols)
-        addSym(d, entry);
-    }
+    drain();
 
     // S_ATTR_LIVE_SUPPORT sections are live if they point _to_ a live
     // section. Process them in a second pass.
@@ -195,12 +258,12 @@ void MarkLiveImpl<RecordWhyLive>::markTransitively() {
             InputSection *referentIsec = nullptr;
             if (auto *d = dyn_cast<Defined>(s))
               referentIsec = d->isec();
-            enqueue(isec, 0, makeEntry(referentIsec, nullptr));
+            enqueue(isec, 0, makeEntry(referentIsec, nullptr), nextLevel);
           }
         } else {
           auto *referentIsec = cast<InputSection *>(r.referent);
           if (referentIsec->isLive(r.getAddend()))
-            enqueue(isec, 0, makeEntry(referentIsec, nullptr));
+            enqueue(isec, 0, makeEntry(referentIsec, nullptr), nextLevel);
         }
       }
     }
@@ -209,7 +272,7 @@ void MarkLiveImpl<RecordWhyLive>::markTransitively() {
     // which in turn could mark additional S_ATTR_LIVE_SUPPORT sections live.
     // Iterate. In practice, the second iteration won't mark additional
     // S_ATTR_LIVE_SUPPORT sections live.
-  } while (!worklist.empty());
+  } while (!nextLevel.empty());
 }
 
 // Set live bit on for each reachable chunk. Unmarked (unreachable)
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index e5f79dc0f8e751..0e79621a9cae97 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -694,7 +694,8 @@ def whyload : Flag<["-"], "whyload">,
     Group<grp_introspect>;
 def why_live : Separate<["-"], "why_live">,
     MetaVarName<"<symbol>">,
-    HelpText<"Log a chain of references to <symbol>, for use with -dead_strip">,
+    HelpText<"Log a chain of references to <symbol>, for use with -dead_strip. "
+             "The chain is deterministic only with --threads=1">,
     Group<grp_introspect>;
 def print_statistics : Flag<["-"], "print_statistics">,
     HelpText<"Log the linker's memory and CPU usage">,
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
index cdeef934752229..29b8079ea94e8a 100644
--- a/lld/MachO/Symbols.h
+++ b/lld/MachO/Symbols.h
@@ -109,7 +109,7 @@ class Symbol {
   bool isUsedInRegularObj : 1;
 
   // True if this symbol is used from a live section.
-  bool used : 1;
+  bool used;
 };
 
 class Defined : public Symbol {
diff --git a/lld/test/MachO/why-live.s b/lld/test/MachO/why-live.s
index 40d94e9da25c65..4561aded459b88 100644
--- a/lld/test/MachO/why-live.s
+++ b/lld/test/MachO/why-live.s
@@ -1,27 +1,27 @@
 # REQUIRES: x86
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.o
-# RUN: %lld -lSystem -dead_strip -why_live _foo -why_live _undef -U _undef \
-# RUN:   -why_live _support -why_live _support_refs_dylib_fun \
+# RUN: %lld -lSystem -dead_strip --threads=1 -why_live _foo -why_live _undef \
+# RUN:   -U _undef -why_live _support -why_live _support_refs_dylib_fun \
 # RUN:   -why_live _abs %t.o -o /dev/null 2>&1 | FileCheck %s
 
 ## Due to an implementation detail, LLD is not able to report -why_live info for
 ## absolute symbols. (ld64 has the same shortcoming.)
 # CHECK-NOT:   _abs
-# CHECK:       _foo from {{.*}}why-live.s.tmp.o
-# CHECK-NEXT:    _quux from {{.*}}why-live.s.tmp.o
-# CHECK-NEXT:  _undef from {{.*}}why-live.s.tmp.o
+# CHECK:       _undef from {{.*}}why-live.s.tmp.o
 # CHECK-NEXT:    _main from {{.*}}why-live.s.tmp.o
-## Our handling of live_support sections can be improved... we should print the
-## dylib symbol that keeps _support_refs_dylib_fun alive, instead of printing
-## the live_support symbol's name itself. (ld64 seems to have the same issue.)
-# CHECK-NEXT: _support_refs_dylib_fun from {{.*}}why-live.s.tmp.o
-# CHECK-NEXT:   _support_refs_dylib_fun from {{.*}}why-live.s.tmp.o
-## Again, this can be improved: we shouldn't be printing _support twice. (ld64
-## seems to have the same issue.)
+# CHECK-NEXT:  _foo from {{.*}}why-live.s.tmp.o
+# CHECK-NEXT:    _main from {{.*}}why-live.s.tmp.o
+## Our handling of live_support sections can be improved: we shouldn't be
+## printing _support twice. (ld64 seems to have the same issue.)
 # CHECK-NEXT:  _support from {{.*}}why-live.s.tmp.o
 # CHECK-NEXT:    _support from {{.*}}why-live.s.tmp.o
 # CHECK-NEXT:      _foo from {{.*}}why-live.s.tmp.o
+## Again, this can be improved... we should print the dylib symbol that keeps
+## _support_refs_dylib_fun alive, instead of printing the live_support symbol's
+## name itself. (ld64 seems to have the same issue.)
+# CHECK-NEXT: _support_refs_dylib_fun from {{.*}}why-live.s.tmp.o
+# CHECK-NEXT:   _support_refs_dylib_fun from {{.*}}why-live.s.tmp.o
 # CHECK-EMPTY:
 
 .text



More information about the llvm-branch-commits mailing list