[lld] [ELF] Parallelize input file loading (PR #191690)

[Peter Smith via llvm-commits]

================
@@ -253,93 +238,59 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
   MemoryBufferRef mbref = *buffer;
 
   if (ctx.arg.formatBinary) {
-    files.push_back(std::make_unique<BinaryFile>(ctx, mbref));
-    if (!isInGroup)
-      ++nextGroupId;
-    return;
-  }
-
-  switch (identify_magic(mbref.getBuffer())) {
-  case file_magic::unknown:
-    readLinkerScript(ctx, mbref);
-    return;
-  case file_magic::archive: {
-    auto members = getArchiveMembers(ctx, mbref);
-    if (inWholeArchive) {
-      for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
-        if (isBitcode(p.first))
-          files.push_back(std::make_unique<BitcodeFile>(ctx, p.first, path,
-                                                        p.second, false));
-        else if (!tryAddFatLTOFile(p.first, path, p.second, false))
-          files.push_back(createObjFile(ctx, p.first, path));
-      }
+    loadJobs.push_back({mbref,
+                        path,
+                        LoadJob::Binary,
+                        false,
+                        false,
+                        false,
+                        false,
+                        nextGroupId,
+                        {},
+                        {}});
+  } else {
+    auto magic = identify_magic(mbref.getBuffer());
+    if (magic == file_magic::unknown) {
+      readLinkerScript(ctx, mbref);
       return;
     }
-
-    archiveFiles.emplace_back(path, members.size());
-
-    // Handle archives and --start-lib/--end-lib using the same code path. This
-    // scans all the ELF relocatable object files and bitcode files in the
-    // archive rather than just the index file, with the benefit that the
-    // symbols are only loaded once. For many projects archives see high
-    // utilization rates and it is a net performance win. --start-lib scans
-    // symbols in the same order that llvm-ar adds them to the index, so in the
-    // common case the semantics are identical. If the archive symbol table was
-    // created in a different order, or is incomplete, this strategy has
-    // different semantics. Such output differences are considered user error.
-    //
-    // All files within the archive get the same group ID to allow mutual
-    // references for --warn-backrefs.
-    SaveAndRestore saved(isInGroup, true);
-    for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
-      auto magic = identify_magic(p.first.getBuffer());
-      if (magic == file_magic::elf_relocatable) {
-        if (!tryAddFatLTOFile(p.first, path, p.second, true))
-          files.push_back(createObjFile(ctx, p.first, path, true));
-      } else if (magic == file_magic::bitcode)
-        files.push_back(
-            std::make_unique<BitcodeFile>(ctx, p.first, path, p.second, true));
-      else
-        Warn(ctx) << path << ": archive member '"
-                  << p.first.getBufferIdentifier()
-                  << "' is neither ET_REL nor LLVM bitcode";
-    }
-    if (!saved.get())
-      ++nextGroupId;
-    return;
-  }
-  case file_magic::elf_shared_object: {
-    if (ctx.arg.isStatic) {
-      ErrAlways(ctx) << "attempted static link of dynamic object " << path;
+    LoadJob::Kind kind;
+    switch (magic) {
+    case file_magic::archive:
+      kind = LoadJob::Archive;
+      break;
+    case file_magic::elf_relocatable:
+      kind = LoadJob::Obj;
+      break;
+    case file_magic::bitcode:
+      kind = LoadJob::Bitcode;
+      break;
+    case file_magic::elf_shared_object:
+      if (ctx.arg.isStatic) {
+        Err(ctx) << "attempted static link of dynamic object " << path;
+        return;
+      }
+      kind = LoadJob::Shared;
+      break;
+    default:
+      Err(ctx) << path << ": unknown file type";
       return;
     }
-
-    // Shared objects are identified by soname. soname is (if specified)
----------------
smithp35 wrote:

We've lost this comment in translation. I think it could be moved to 2174.

https://github.com/llvm/llvm-project/pull/191690