[llvm] [DTLTO][LLVM] Integrated Distributed ThinLTO (DTLTO) (PR #127749)
Teresa Johnson via llvm-commits
llvm-commits at lists.llvm.org
Fri May 16 09:54:08 PDT 2025
================
@@ -2142,3 +2182,322 @@ std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
});
return ModulesOrdering;
}
+
+namespace {
+// For this out-of-process backend no codegen is done when invoked for each
+// task. Instead we generate the required information (e.g. the summary index
+// shard, import list, etc..) to allow for the codegen to be performed
+// externally (similar to WriteIndexesThinBackend). This backend's `wait`
+// function then invokes an external distributor process to do backend
+// compilations.
+class OutOfProcessThinBackend : public CGThinBackend {
+ using SString = SmallString<128>;
+
+ BumpPtrAllocator Alloc;
+ StringSaver Saver{Alloc};
+
+ SString LinkerOutputFile;
+
+ SString DistributorPath;
+ ArrayRef<StringRef> DistributorArgs;
+
+ SString RemoteCompiler;
+ ArrayRef<StringRef> RemoteCompilerArgs;
+
+ bool SaveTemps;
+
+ SmallVector<StringRef, 0> CodegenOptions;
+ DenseSet<StringRef> CommonInputs;
+
+ // Information specific to individual backend compilation job.
+ struct Job {
+ unsigned Task;
+ StringRef ModuleID;
+ StringRef NativeObjectPath;
+ StringRef SummaryIndexPath;
+ ImportsFilesContainer ImportFiles;
+ };
+ // The set of backend compilations jobs.
+ SmallVector<Job> Jobs;
+
+ // A unique string to identify the current link.
+ SmallString<8> UID;
+
+ // The offset to the first ThinLTO task.
+ unsigned ThinLTOTaskOffset;
+
+ // The target triple to supply for backend compilations.
+ StringRef Triple;
+
+public:
+ OutOfProcessThinBackend(
+ const Config &Conf, ModuleSummaryIndex &CombinedIndex,
+ ThreadPoolStrategy ThinLTOParallelism,
+ const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
+ AddStreamFn AddStream, lto::IndexWriteCallback OnWrite,
+ bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
+ StringRef LinkerOutputFile, StringRef Distributor,
+ ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,
+ ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps)
+ : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
+ AddStream, OnWrite, ShouldEmitIndexFiles,
+ ShouldEmitImportsFiles, ThinLTOParallelism),
+ LinkerOutputFile(LinkerOutputFile), DistributorPath(Distributor),
+ DistributorArgs(DistributorArgs), RemoteCompiler(RemoteCompiler),
+ RemoteCompilerArgs(RemoteCompilerArgs), SaveTemps(SaveTemps) {}
+
+ virtual void setup(unsigned ThinLTONumTasks, unsigned ThinLTOTaskOffset,
+ StringRef Triple) override {
+ UID = itostr(sys::Process::getProcessId());
+ Jobs.resize((size_t)ThinLTONumTasks);
+ this->ThinLTOTaskOffset = ThinLTOTaskOffset;
+ this->Triple = Triple;
+ }
+
+ Error start(
+ unsigned Task, BitcodeModule BM,
+ const FunctionImporter::ImportMapTy &ImportList,
+ const FunctionImporter::ExportSetTy &ExportList,
+ const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
+ MapVector<StringRef, BitcodeModule> &ModuleMap) override {
+
+ StringRef ModulePath = BM.getModuleIdentifier();
+
+ SString ObjFilePath = sys::path::parent_path(LinkerOutputFile);
+ sys::path::append(ObjFilePath, sys::path::stem(ModulePath) + "." +
+ itostr(Task) + "." + UID + ".native.o");
+
+ Job &J = Jobs[Task - ThinLTOTaskOffset];
+ J = {Task,
+ ModulePath,
+ Saver.save(ObjFilePath.str()),
+ Saver.save(ObjFilePath.str() + ".thinlto.bc"),
+ {}};
+
+ assert(ModuleToDefinedGVSummaries.count(ModulePath));
+ BackendThreadPool.async(
+ [=](Job &J, const FunctionImporter::ImportMapTy &ImportList) {
+ if (auto E = emitFiles(ImportList, J.ModuleID, J.SummaryIndexPath,
+ J.ModuleID.str(), J.ImportFiles)) {
+ std::unique_lock<std::mutex> L(ErrMu);
+ if (Err)
+ Err = joinErrors(std::move(*Err), std::move(E));
+ else
+ Err = std::move(E);
+ }
+ },
+ std::ref(J), std::ref(ImportList));
+
+ return Error::success();
+ }
+
+ // Derive a set of Clang options that will be shared/common for all DTLTO
+ // backend compilations. We are intentionally minimal here as these options
+ // must remain synchronized with the behavior of Clang. DTLTO does not support
+ // all the features available with in-process LTO. More features are expected
+ // to be added over time. Users can specify Clang options directly if a
+ // feature is not supported. Note that explicitly specified options that imply
+ // additional input or output file dependencies must be communicated to the
+ // distribution system, potentially by setting extra options on the
+ // distributor program.
+ void buildCommonRemoteCompilerOptions() {
+ const lto::Config &C = Conf;
+ auto &Ops = CodegenOptions;
+ llvm::Triple TT{Triple};
+
+ Ops.push_back(Saver.save("-O" + Twine(C.OptLevel)));
+
+ if (C.Options.EmitAddrsig)
+ Ops.push_back("-faddrsig");
+ if (C.Options.FunctionSections)
+ Ops.push_back("-ffunction-sections");
+ if (C.Options.DataSections)
+ Ops.push_back("-fdata-sections");
+
+ if (C.RelocModel == Reloc::PIC_)
+ // Clang doesn't have -fpic for all triples.
+ if (!TT.isOSBinFormatCOFF())
+ Ops.push_back("-fpic");
+
+ // Turn on/off warnings about profile cfg mismatch (default on)
+ // --lto-pgo-warn-mismatch.
+ if (!C.PGOWarnMismatch) {
+ Ops.push_back("-mllvm");
+ Ops.push_back("-no-pgo-warn-mismatch");
+ }
+
+ // Enable sample-based profile guided optimizations.
+ // Sample profile file path --lto-sample-profile=<value>.
+ if (!C.SampleProfile.empty()) {
+ Ops.push_back(
+ Saver.save("-fprofile-sample-use=" + Twine(C.SampleProfile)));
+ CommonInputs.insert(C.SampleProfile);
+ }
+
+ // We don't know which of options will be used by Clang.
+ Ops.push_back("-Wno-unused-command-line-argument");
+
+ // Forward any supplied options.
+ if (!RemoteCompilerArgs.empty())
+ for (auto &a : RemoteCompilerArgs)
+ Ops.push_back(a);
+ }
+
+ // Generates a JSON file describing the backend compilations, for the
+ // distributor.
+ bool emitDistributorJson(StringRef DistributorJson) {
+ using json::Array;
+ std::error_code EC;
+ raw_fd_ostream OS(DistributorJson, EC);
+ if (EC)
+ return false;
+
+ json::OStream JOS(OS);
+ JOS.object([&]() {
+ // Information common to all jobs.
+ JOS.attributeObject("common", [&]() {
+ JOS.attribute("linker_output", LinkerOutputFile);
+
+ JOS.attributeArray("args", [&]() {
+ JOS.value(RemoteCompiler);
+
+ JOS.value("-c");
+
+ JOS.value(Saver.save("--target=" + Twine(Triple)));
+
+ for (const auto &A : CodegenOptions)
+ JOS.value(A);
+ });
+
+ JOS.attribute("inputs", Array(CommonInputs));
+ });
+
+ // Per-compilation-job information.
+ JOS.attributeArray("jobs", [&]() {
+ for (const auto &J : Jobs) {
+ assert(J.Task != 0);
+
+ SmallVector<StringRef, 2> Inputs;
+ SmallVector<StringRef, 1> Outputs;
+
+ JOS.object([&]() {
+ JOS.attributeArray("args", [&]() {
+ JOS.value(J.ModuleID);
+ Inputs.push_back(J.ModuleID);
+
+ JOS.value(
+ Saver.save("-fthinlto-index=" + Twine(J.SummaryIndexPath)));
+ Inputs.push_back(J.SummaryIndexPath);
+
+ JOS.value("-o");
+ JOS.value(J.NativeObjectPath);
+ Outputs.push_back(J.NativeObjectPath);
+ });
+
+ // Add the bitcode files from which imports will be made. These do
+ // not explicitly appear on the backend compilation command lines
+ // but are recorded in the summary index shards.
+ llvm::append_range(Inputs, J.ImportFiles);
+ JOS.attribute("inputs", Array(Inputs));
+
+ JOS.attribute("outputs", Array(Outputs));
+ });
+ }
+ });
+ });
+
+ return true;
+ }
+
+ void removeFile(StringRef FileName) {
+ std::error_code EC = sys::fs::remove(FileName, true);
+ if (EC && EC != std::make_error_code(std::errc::no_such_file_or_directory))
+ errs() << "warning: could not remove the file '" << FileName
+ << "': " << EC.message() << "\n";
+ }
+
+ Error wait() override {
+ // Wait for the information on the required backend compilations to be
+ // gathered.
+ BackendThreadPool.wait();
+ if (Err)
+ return std::move(*Err);
+
+ auto CleanPerJobFiles = llvm::make_scope_exit([&] {
+ if (!SaveTemps)
+ for (auto &Job : Jobs) {
+ removeFile(Job.NativeObjectPath);
+ if (!ShouldEmitIndexFiles)
+ removeFile(Job.SummaryIndexPath);
+ }
+ });
+
+ const StringRef BCError = "DTLTO backend compilation: ";
+
+ buildCommonRemoteCompilerOptions();
+
+ SString JsonFile = sys::path::parent_path(LinkerOutputFile);
+ sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID +
+ ".dist-file.json");
+ if (!emitDistributorJson(JsonFile))
+ return make_error<StringError>(
+ BCError + "failed to generate distributor JSON script: " + JsonFile,
+ inconvertibleErrorCode());
+ auto CleanJson = llvm::make_scope_exit([&] {
+ if (!SaveTemps)
+ removeFile(JsonFile);
+ });
+
+ SmallVector<StringRef, 3> Args = {DistributorPath};
+ llvm::append_range(Args, DistributorArgs);
+ Args.push_back(JsonFile);
+ std::string ErrMsg;
+ if (sys::ExecuteAndWait(Args[0], Args,
+ /*Env=*/std::nullopt, /*Redirects=*/{},
+ /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg)) {
+ return make_error<StringError>(
+ BCError + "distributor execution failed" +
+ (!ErrMsg.empty() ? ": " + ErrMsg + Twine(".") : Twine(".")),
+ inconvertibleErrorCode());
+ }
+
+ for (auto &Job : Jobs) {
+ // Load the native object from a file into a memory buffer
+ // and store its contents in the output buffer.
+ auto ObjFileMbOrErr =
+ MemoryBuffer::getFile(Job.NativeObjectPath, false, false);
----------------
teresajohnson wrote:
nit: document constant args
https://github.com/llvm/llvm-project/pull/127749
More information about the llvm-commits
mailing list