[lld] r365800 - [lld-link] implement -thinlto-index-only
Bob Haarman via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 11 11:03:15 PDT 2019
Author: inglorion
Date: Thu Jul 11 11:03:14 2019
New Revision: 365800
URL: http://llvm.org/viewvc/llvm-project?rev=365800&view=rev
Log:
[lld-link] implement -thinlto-index-only
Summary:
This implements -thinlto-index-only, -thinlto-index-only:,
and -thinlto-emit-imports-files options in lld-link. They are
analogous to their counterparts in ld.lld: -thinlto-index-only
causes us to perform ThinLTO's thin link and write index files,
but not perform code generation. -thinlto-index-only: does the
same, but also writes a text file listing the native object
files expected to be generated. -thinlto-emit-imports-files
creates a text file next to each index file, listing the files
to import from.
Reviewers: ruiu, tejohnson, pcc, rnk
Subscribers: mehdi_amini, steven_wu, dexonsmith, arphaman, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D64461
Added:
lld/trunk/test/COFF/Inputs/thinlto-empty.ll
lld/trunk/test/COFF/Inputs/thinlto.ll
lld/trunk/test/COFF/thinlto-emit-imports.ll
lld/trunk/test/COFF/thinlto-index-only.ll
Modified:
lld/trunk/COFF/Config.h
lld/trunk/COFF/Driver.cpp
lld/trunk/COFF/LTO.cpp
lld/trunk/COFF/LTO.h
lld/trunk/COFF/Options.td
Modified: lld/trunk/COFF/Config.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/Config.h?rev=365800&r1=365799&r2=365800&view=diff
==============================================================================
--- lld/trunk/COFF/Config.h (original)
+++ lld/trunk/COFF/Config.h Thu Jul 11 11:03:14 2019
@@ -179,6 +179,9 @@ struct Configuration {
// Used for /lldmap.
std::string mapFile;
+ // Used for /thinlto-index-only:
+ llvm::StringRef thinLTOIndexOnlyArg;
+
uint64_t imageBase = -1;
uint64_t fileAlign = 512;
uint64_t stackReserve = 1024 * 1024;
@@ -209,6 +212,8 @@ struct Configuration {
bool repro = false;
bool swaprunCD = false;
bool swaprunNet = false;
+ bool thinLTOEmitImportsFiles;
+ bool thinLTOIndexOnly;
};
extern Configuration *config;
Modified: lld/trunk/COFF/Driver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/Driver.cpp?rev=365800&r1=365799&r2=365800&view=diff
==============================================================================
--- lld/trunk/COFF/Driver.cpp (original)
+++ lld/trunk/COFF/Driver.cpp Thu Jul 11 11:03:14 2019
@@ -1441,6 +1441,11 @@ void LinkerDriver::link(ArrayRef<const c
fatal("/manifestinput: requires /manifest:embed");
}
+ config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
+ config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+ args.hasArg(OPT_thinlto_index_only_arg);
+ config->thinLTOIndexOnlyArg =
+ args.getLastArgValue(OPT_thinlto_index_only_arg);
// Handle miscellaneous boolean flags.
config->allowBind = args.hasFlag(OPT_allowbind, OPT_allowbind_no, true);
config->allowIsolation =
@@ -1727,8 +1732,18 @@ void LinkerDriver::link(ArrayRef<const c
return;
// Do LTO by compiling bitcode input files to a set of native COFF files then
- // link those files.
+ // link those files (unless -thinlto-index-only was given, in which case we
+ // resolve symbols and write indices, but don't generate native code or link).
symtab->addCombinedLTOObjects();
+
+ // If -thinlto-index-only is given, we should create only "index
+ // files" and not object files. Index file creation is already done
+ // in addCombinedLTOObject, so we are done if that's the case.
+ if (config->thinLTOIndexOnly)
+ return;
+
+ // If we generated native object files from bitcode files, this resolves
+ // references to the symbols we use from them.
run();
if (args.hasArg(OPT_include_optional)) {
Modified: lld/trunk/COFF/LTO.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/LTO.cpp?rev=365800&r1=365799&r2=365800&view=diff
==============================================================================
--- lld/trunk/COFF/LTO.cpp (original)
+++ lld/trunk/COFF/LTO.cpp Thu Jul 11 11:03:14 2019
@@ -18,6 +18,7 @@
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
@@ -41,7 +42,19 @@ using namespace llvm::object;
using namespace lld;
using namespace lld::coff;
-static std::unique_ptr<lto::LTO> createLTO() {
+// Creates an empty file to and returns a raw_fd_ostream to write to it.
+static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) {
+ std::error_code ec;
+ auto ret =
+ llvm::make_unique<raw_fd_ostream>(file, ec, sys::fs::OpenFlags::F_None);
+ if (ec) {
+ error("cannot open " + file + ": " + ec.message());
+ return nullptr;
+ }
+ return ret;
+}
+
+static lto::Config createConfig() {
lto::Config c;
c.Options = initTargetOptionsFromCodeGenFlags();
@@ -67,14 +80,27 @@ static std::unique_ptr<lto::LTO> createL
if (config->saveTemps)
checkError(c.addSaveTemps(std::string(config->outputFile) + ".",
/*UseInputModulePath*/ true));
+ return c;
+}
+
+BitcodeCompiler::BitcodeCompiler() {
+ // Initialize indexFile.
+ if (!config->thinLTOIndexOnlyArg.empty())
+ indexFile = openFile(config->thinLTOIndexOnlyArg);
+
+ // Initialize ltoObj.
lto::ThinBackend backend;
- if (config->thinLTOJobs != 0)
+ if (config->thinLTOIndexOnly) {
+ auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); };
+ backend = lto::createWriteIndexesThinBackend(
+ "", "", config->thinLTOEmitImportsFiles, indexFile.get(), OnIndexWrite);
+ } else if (config->thinLTOJobs != 0) {
backend = lto::createInProcessThinBackend(config->thinLTOJobs);
- return llvm::make_unique<lto::LTO>(std::move(c), backend,
- config->ltoPartitions);
-}
+ }
-BitcodeCompiler::BitcodeCompiler() : ltoObj(createLTO()) {}
+ ltoObj = llvm::make_unique<lto::LTO>(createConfig(), backend,
+ config->ltoPartitions);
+}
BitcodeCompiler::~BitcodeCompiler() = default;
@@ -86,6 +112,9 @@ void BitcodeCompiler::add(BitcodeFile &f
std::vector<Symbol *> symBodies = f.getSymbols();
std::vector<lto::SymbolResolution> resols(symBodies.size());
+ if (config->thinLTOIndexOnly)
+ thinIndices.insert(obj.getName());
+
// Provide a resolution to the LTO API for each symbol.
for (const lto::InputFile::Symbol &objSym : obj.symbols()) {
Symbol *sym = symBodies[symNum];
@@ -129,6 +158,23 @@ std::vector<StringRef> BitcodeCompiler::
},
cache));
+ // Emit empty index files for non-indexed files
+ for (StringRef S : thinIndices) {
+ std::string Path(S);
+ openFile(Path + ".thinlto.bc");
+ if (config->thinLTOEmitImportsFiles)
+ openFile(Path + ".imports");
+ }
+
+ // ThinLTO with index only option is required to generate only the index
+ // files. After that, we exit from linker and ThinLTO backend runs in a
+ // distributed environment.
+ if (config->thinLTOIndexOnly) {
+ if (indexFile)
+ indexFile->close();
+ return {};
+ }
+
if (!config->ltoCache.empty())
pruneCache(config->ltoCache, config->ltoCachePolicy);
Modified: lld/trunk/COFF/LTO.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/LTO.h?rev=365800&r1=365799&r2=365800&view=diff
==============================================================================
--- lld/trunk/COFF/LTO.h (original)
+++ lld/trunk/COFF/LTO.h Thu Jul 11 11:03:14 2019
@@ -21,7 +21,9 @@
#define LLD_COFF_LTO_H
#include "lld/Common/LLVM.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <vector>
@@ -49,6 +51,8 @@ private:
std::unique_ptr<llvm::lto::LTO> ltoObj;
std::vector<SmallString<0>> buf;
std::vector<std::unique_ptr<MemoryBuffer>> files;
+ std::unique_ptr<llvm::raw_fd_ostream> indexFile;
+ llvm::DenseSet<StringRef> thinIndices;
};
}
}
Modified: lld/trunk/COFF/Options.td
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/Options.td?rev=365800&r1=365799&r2=365800&view=diff
==============================================================================
--- lld/trunk/COFF/Options.td (original)
+++ lld/trunk/COFF/Options.td Thu Jul 11 11:03:14 2019
@@ -176,6 +176,15 @@ def pdb_source_path : P<"pdbsourcepath",
"Base path used to make relative source file path absolute in PDB">;
def rsp_quoting : Joined<["--"], "rsp-quoting=">,
HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">;
+def thinlto_emit_imports_files :
+ F<"thinlto-emit-imports-files">,
+ HelpText<"Emit .imports files with -thinlto-index-only">;
+def thinlto_index_only :
+ F<"thinlto-index-only">,
+ HelpText<"Instead of linking, emit ThinLTO index files">;
+def thinlto_index_only_arg : P<
+ "thinlto-index-only",
+ "-thinlto-index-only and also write native module names to file">;
def dash_dash_version : Flag<["--"], "version">,
HelpText<"Print version information">;
defm threads: B<"threads",
Added: lld/trunk/test/COFF/Inputs/thinlto-empty.ll
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/thinlto-empty.ll?rev=365800&view=auto
==============================================================================
--- lld/trunk/test/COFF/Inputs/thinlto-empty.ll (added)
+++ lld/trunk/test/COFF/Inputs/thinlto-empty.ll Thu Jul 11 11:03:14 2019
@@ -0,0 +1,2 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
Added: lld/trunk/test/COFF/Inputs/thinlto.ll
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/Inputs/thinlto.ll?rev=365800&view=auto
==============================================================================
--- lld/trunk/test/COFF/Inputs/thinlto.ll (added)
+++ lld/trunk/test/COFF/Inputs/thinlto.ll Thu Jul 11 11:03:14 2019
@@ -0,0 +1,6 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+define void @g() {
+ ret void
+}
Added: lld/trunk/test/COFF/thinlto-emit-imports.ll
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/thinlto-emit-imports.ll?rev=365800&view=auto
==============================================================================
--- lld/trunk/test/COFF/thinlto-emit-imports.ll (added)
+++ lld/trunk/test/COFF/thinlto-emit-imports.ll Thu Jul 11 11:03:14 2019
@@ -0,0 +1,59 @@
+; REQUIRES: x86
+
+; Generate summary sections and test lld handling.
+; RUN: opt -module-summary %s -o %t1.obj
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.obj
+
+; Include a file with an empty module summary index, to ensure that the expected
+; output files are created regardless, for a distributed build system.
+; RUN: opt -module-summary %p/Inputs/thinlto-empty.ll -o %t3.obj
+
+; Ensure lld generates imports files if requested for distributed backends.
+; RUN: rm -f %t3.obj.imports %t3.obj.thinlto.bc
+; RUN: lld-link -entry:main -thinlto-index-only \
+; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj -out:%t4.exe
+
+; The imports file for this module contains the bitcode file for
+; Inputs/thinlto.ll
+; RUN: cat %t1.obj.imports | count 1
+; RUN: cat %t1.obj.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: thinlto-emit-imports.ll.tmp2.obj
+
+; The imports file for Input/thinlto.ll is empty as it does not import anything.
+; RUN: cat %t2.obj.imports | count 0
+
+; The imports file for Input/thinlto_empty.ll is empty but should exist.
+; RUN: cat %t3.obj.imports | count 0
+
+; The index file should be created even for the input with an empty summary.
+; RUN: ls %t3.obj.thinlto.bc
+
+; Ensure lld generates error if unable to write to imports file.
+; RUN: rm -f %t3.obj.imports
+; RUN: touch %t3.obj.imports
+; RUN: chmod 400 %t3.obj.imports
+; RUN: not lld-link -entry:main -thinlto-index-only \
+; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj \
+; RUN: -out:%t4.exe 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: cannot open {{.*}}3.obj.imports: {{P|p}}ermission denied
+
+; Ensure lld doesn't generate import files when thinlto-index-only is not enabled
+; RUN: rm -f %t1.obj.imports
+; RUN: rm -f %t2.obj.imports
+; RUN: rm -f %t3.obj.imports
+; RUN: lld-link -entry:main -thinlto-emit-imports-files \
+; RUN: %t1.obj %t2.obj %t3.obj -out:%t4.exe
+; RUN: not ls %t1.obj.imports
+; RUN: not ls %t2.obj.imports
+; RUN: not ls %t3.obj.imports
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+declare void @g(...)
+
+define void @main() {
+entry:
+ call void (...) @g()
+ ret void
+}
Added: lld/trunk/test/COFF/thinlto-index-only.ll
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/thinlto-index-only.ll?rev=365800&view=auto
==============================================================================
--- lld/trunk/test/COFF/thinlto-index-only.ll (added)
+++ lld/trunk/test/COFF/thinlto-index-only.ll Thu Jul 11 11:03:14 2019
@@ -0,0 +1,52 @@
+; REQUIRES: x86
+
+; Basic ThinLTO tests.
+; RUN: opt -thinlto-bc %s -o %t1.obj
+; RUN: opt -thinlto-bc %p/Inputs/thinlto.ll -o %t2.obj
+; RUN: opt -thinlto-bc %p/Inputs/thinlto-empty.ll -o %t3.obj
+
+; Ensure lld generates an index and not a binary if requested.
+; RUN: rm -f %t4.exe
+; RUN: lld-link -thinlto-index-only -entry:main %t1.obj %t2.obj -out:%t4.exe
+; RUN: llvm-bcanalyzer -dump %t1.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump %t2.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: not test -e %t4.exe
+
+; The backend index for this module contains summaries from itself and
+; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.obj'
+; BACKEND1: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.obj'
+; BACKEND1-NOT: <ENTRY
+; BACKEND1: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-5300342847281564238|-2624081020897602054}}
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-5300342847281564238|-2624081020897602054}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+; The backend index for Input/thinlto.ll contains summaries from itself only,
+; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp2.obj'
+; BACKEND2-NOT: <ENTRY
+; BACKEND2: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID op0=1 op1=-5300342847281564238
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+declare void @g(...)
+
+define void @main() {
+ call void (...) @g()
+ ret void
+}
More information about the llvm-commits
mailing list