[clang] [lld] [llvm] [DTLTO] Refactor the DTLTO code. (PR #192629)

[Konstantin Belochapka via cfe-commits]

================
@@ -1,83 +1,407 @@
-//===- DTLTO.h - Distributed ThinLTO functions and classes ----*- C++ -*-===//
+//===- DTLTO.h - Integrated Distributed ThinLTO implementation ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===---------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+//
+// \file
+// Declarations for Integrated Distributed ThinLTO, including the DTLTO class
+// and the distribution driver. The implementation focuses on preparing input
+// files for distribution.
+//
+//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_DTLTO_DTLTO_H
 #define LLVM_DTLTO_DTLTO_H
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/LTO/LTO.h"
-#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Signals.h"
+
+#include <functional>
+#include <vector>
 
 namespace llvm {
 namespace lto {
 
-// The purpose of this class is to prepare inputs so that distributed ThinLTO
-// backend compilations can succeed.
-//
-// For distributed compilation, each input must exist as an individual bitcode
-// file on disk and be loadable via its ModuleID. This requirement is not met
-// for archive members, as an archive is a collection of files rather than a
-// standalone file. Similarly, for FatLTO objects, the bitcode is stored in a
-// section of the containing ELF object file. To address this, the class ensures
-// that an individual bitcode file exists for each input (by writing it out if
-// necessary) and that the ModuleID is updated to point to it. Module IDs are
-// also normalized on Windows to remove short 8.3 form paths that cannot be
-// loaded on remote machines.
-//
-// The class ensures that lto::InputFile objects are preserved until enough of
-// the LTO pipeline has executed to determine the required per-module
-// information, such as whether a module will participate in ThinLTO.
+/// Prepares inputs for Distributed ThinLTO so that backend compilations can use
+/// individual bitcode paths and consistent module IDs.
+///
+/// Each input must exist as an individual bitcode file on disk and be loadable
+/// via its ModuleID. Archive members and FatLTO objects do not satisfy that by
+/// default; this class writes bitcode out when needed and updates ModuleID.
+/// On Windows, module IDs are normalized to remove short 8.3 path components
+/// that are machine-local and break distribution; other normalization is left
+/// to DTLTO distributors.
+///
+/// Input files are kept until the pipeline has determined per-module ThinLTO
+/// participation. addInput() performs: (1) register the input; (2) on Windows,
+/// normalize module ID for standalone bitcode; (3) for thin archive members,
+/// set module ID to the on-disk member path; (4) for other archives and FatLTO,
+/// set module ID to a unique path and serialize content in
+/// serializeBitcodeArchiveMembers().
 class DTLTO : public LTO {
   using Base = LTO;
 
 public:
-  LLVM_ABI DTLTO(Config Conf, ThinBackend Backend,
-                 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode,
-                 StringRef LinkerOutputFile, bool SaveTemps)
-      : Base(std::move(Conf), Backend, ParallelCodeGenParallelismLevel,
-             LTOMode),
-        LinkerOutputFile(LinkerOutputFile), SaveTemps(SaveTemps) {
+  LLVM_ABI DTLTO(Config Conf, unsigned ParallelCodeGenParallelismLevel,
+                 LTOKind LTOMode, IndexWriteCallback OnWrite,
+                 bool EmitIndexFiles, bool EmitImportsFiles,
+                 StringRef LinkerOutputFile, StringRef Distributor,
+                 ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,
+                 ArrayRef<StringRef> RemoteCompilerPrependArgs,
+                 ArrayRef<StringRef> RemoteCompilerArgs,
+                 AddBufferFn AddBufferArg, bool SaveTempsArg)
+      : Base(std::move(Conf), writeIndexesBackendInstance(),
+             ParallelCodeGenParallelismLevel, LTOMode),
+        AddBuffer(AddBufferArg), SaveTemps(SaveTempsArg),
+        ShouldEmitIndexFiles(EmitIndexFiles),
+        ShouldEmitImportFiles(EmitImportsFiles), OnIndexWriteCb(OnWrite),
+        DistributorParams{Distributor,        DistributorArgs,
+                          RemoteCompiler,     RemoteCompilerPrependArgs,
+                          RemoteCompilerArgs, LinkerOutputFile} {
     assert(!LinkerOutputFile.empty() && "expected a valid linker output file");
   }
 
-  // Add an input file and prepare it for distribution.
+  // Create an instance of WriteIndexesBackend class.
+  static lto::ThinBackend writeIndexesBackendInstance() {
+    return lto::createWriteIndexesThinBackend(hardware_concurrency(), "", "",
+                                              "", true, nullptr, nullptr);
+  }
+
+  /// Add an input file and prepare it for distribution.
+  ///
+  /// This function performs the following tasks:
+  /// 1. Add the input file to the LTO object's list of input files.
+  /// 2. For individual bitcode file inputs on Windows only, overwrite the
+  ///    module ID with a normalized path to remove short 8.3 form components.
+  /// 3. For thin archive members, overwrite the module ID with the path
+  ///    (normalized on Windows) to the member file on disk.
+  /// 4. For archive members and FatLTO objects, overwrite the module ID with a
+  ///    unique path (normalized on Windows) naming a file that will contain the
+  ///    member content. The file is created and populated later (see
+  ///    serializeInputs()).
   LLVM_ABI Expected<std::shared_ptr<InputFile>>
   addInput(std::unique_ptr<InputFile> InputPtr) override;
 
-protected:
-  // Save the contents of ThinLTO-enabled input files that must be serialized
-  // for distribution, such as archive members and FatLTO objects, to individual
-  // bitcode files named after the module ID.
-  LLVM_ABI llvm::Error serializeInputsForDistribution() override;
+  /// Runs the DTLTO pipeline. This function calls the supplied AddStream
+  /// function to add native object files to the link.
+  ///
+  /// The Cache parameter is optional. If supplied, it will be used to cache
+  /// native object files and add them to the link.
+  ///
+  /// The client will receive at most one callback (via either AddStream or
+  /// Cache) for each task identifier.
+  LLVM_ABI virtual Error run(AddStreamFn AddStream,
+                             FileCache Cache = {}) override;
+
+private:
+  /// DTLTO archives support.
+  ///
+  /// Save the contents of ThinLTO-enabled input files that must be serialized
+  /// for distribution, such as archive members and FatLTO objects, to
+  /// individual bitcode files named after the module ID.
+  ///
+  /// Must be called after all input files are added but before optimization
+  /// begins. If a file with that name already exists, it is likely a leftover
+  /// from a previously terminated linker process and can be safely overwritten.
+  LLVM_ABI Error serializeBitcodeArchiveMembers();
 
+  // Remove temporary files created to enable distribution.
   LLVM_ABI void cleanup() override;
 
+public:
+  // Mutable and const accessors to the LTO configuration object.
+  Config &getConfig() { return Conf; }
+  const Config &getConfig() const { return Conf; }
+
+  // Set the LTO kind.
+  void setLTOMode(LTOKind Knd) { LTOMode = Knd; }
+  // Replace the ThinLTO backend (e.g. WriteIndexesThinBackend for the thin
+  // link).
+  void setThinBackend(ThinBackend Backend) { ThinLTO.Backend = Backend; }
----------------
kbelochapka wrote:

Removed `setThinBackend()`
Removed `setLTOMode()`, `LTOMode' variable initialization moved to `DTLTO` class constructor.
Replaced `ImportsFilesLists[Task]` with ` J.ImportsFiles`
Renamed `handleArchiveInputs()` into `serializeLTOInputs()`

https://github.com/llvm/llvm-project/pull/192629