[lld] [lld][MachO]Multi-threaded i/o. Twice as fast linking a large project. (PR #147134)

[John Holdsworth via llvm-commits]

================
@@ -282,11 +284,85 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
           ": Archive::children failed: " + toString(std::move(e)));
 }
 
-static InputFile *addFile(StringRef path, LoadType loadType,
-                          bool isLazy = false, bool isExplicit = true,
-                          bool isBundleLoader = false,
-                          bool isForceHidden = false) {
-  std::optional<MemoryBufferRef> buffer = readFile(path);
+class DeferredFile {
+public:
+  StringRef path;
+  bool isLazy;
+  MemoryBufferRef buffer;
+};
+using DeferredFiles = std::vector<DeferredFile>;
+
+// Most input files have been mapped but not yet paged in.
+// This code forces the page-ins on multiple threads so
+// the process is not stalled waiting on disk buffer i/o.
+void multiThreadedPageInBackground(const DeferredFiles &deferred) {
+  static size_t pageSize = Process::getPageSizeEstimate(), totalBytes;
+  static std::mutex mutex;
+  size_t index = 0;
+
+  parallelFor(0, config->readThreads, [&](size_t I) {
+    while (true) {
+      mutex.lock();
+      if (index >= deferred.size()) {
+        mutex.unlock();
+        return;
+      }
+      const StringRef &buff = deferred[index].buffer.getBuffer();
+      totalBytes += buff.size();
+      index += 1;
+      mutex.unlock();
+
+      // Reference each page to load it into memory.
+      for (const char *page = buff.data(), *end = page + buff.size();
+           page < end; page += pageSize)
+        volatile char t = *page;
+    }
+  });
+
+  if (getenv("LLD_MULTI_THREAD_PAGE"))
+    llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
+                 << deferred.size() << "\n";
+}
+
+static void multiThreadedPageIn(const DeferredFiles &deferred) {
+  static std::deque<DeferredFiles *> queue;
+  static std::thread *running;
+  static std::mutex mutex;
+
+  mutex.lock();
+  if (running && (queue.empty() || deferred.empty())) {
+    running->join();
+    delete running;
+    running = nullptr;
+  }
+
+  if (!deferred.empty()) {
+    queue.emplace_back(new DeferredFiles(deferred));
+    if (!running)
+      running = new std::thread([&]() {
+        while (true) {
+          mutex.lock();
+          if (queue.empty()) {
+            mutex.unlock();
+            return;
+          }
+          DeferredFiles *deferred = queue.front();
+          queue.pop_front();
+          mutex.unlock();
+          multiThreadedPageInBackground(*deferred);
+          delete deferred;
+        }
+      });
+  }
+  mutex.unlock();
+}
----------------
johnno1962 wrote:

I'm sorry I'm beginning to miss your comments on the PR so I have to check by going back though emails. I agree if we can find an abstraction that isn't more complicate to use than what we have we should use it. We can't be the first people that have had the requirement to want to start background pieces of work in a controlled serial way.

https://github.com/llvm/llvm-project/pull/147134