[lld] 2638aaf - [LLD][ThinLTO] Add --thinlto-single-module to allow compiling partial modules.
Hongtao Yu via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 15:33:39 PDT 2020
Author: Hongtao Yu
Date: 2020-06-10T15:32:30-07:00
New Revision: 2638aafe1203f64983017f8998fb09ce986a1b28
URL: https://github.com/llvm/llvm-project/commit/2638aafe1203f64983017f8998fb09ce986a1b28
DIFF: https://github.com/llvm/llvm-project/commit/2638aafe1203f64983017f8998fb09ce986a1b28.diff
LOG: [LLD][ThinLTO] Add --thinlto-single-module to allow compiling partial modules.
This change introduces an LLD switch --thinlto-single-module to allow compiling only a part of the input modules. This is specifically enables:
1. Fast investigating/debugging modules of interest without spending time on compiling unrelated modules.
2. Compiler debug dump with -mllvm -debug-only= for specific modules.
It will be useful for large applications which has 1K+ input modules for thinLTO.
The switch can be combined with `--lto-obj-path=` or `--lto-emit-asm` to obtain intermediate object files or assembly files. So far the module name matching is implemented as a fuzzy name lookup where the modules with name containing the switch value are compiled.
E.g,
Command:
ld.lld main.o thin.a --thinlto-single-module=thin.a --lto-obj-path=single.o
log:
[ThinLTO] Selecting thin.a(thin1.o at 168) to compile
[ThinLTO] Selecting thin.a(thin2.o at 228) to compile
Command:
ld.lld main.o thin.a --thinlto-single-module=thin1.o --lto-obj-path=single.o
log:
[ThinLTO] Selecting thin.a(thin1.o at 168) to compile
Differential Revision: https://reviews.llvm.org/D80406
Added:
lld/test/ELF/lto/thinlto-single-module.ll
Modified:
lld/ELF/Config.h
lld/ELF/Driver.cpp
lld/ELF/LTO.cpp
lld/ELF/Options.td
llvm/include/llvm/LTO/Config.h
llvm/include/llvm/LTO/LTO.h
llvm/lib/LTO/LTO.cpp
Removed:
################################################################################
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index cc4547bb0ae2..cfdd1ebf0b03 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -125,6 +125,7 @@ struct Configuration {
std::vector<llvm::StringRef> filterList;
std::vector<llvm::StringRef> searchPaths;
std::vector<llvm::StringRef> symbolOrderingFile;
+ std::vector<llvm::StringRef> thinLTOModulesToCompile;
std::vector<llvm::StringRef> undefined;
std::vector<SymbolVersion> dynamicList;
std::vector<uint8_t> buildIdVector;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index bcef615f1c60..78a60147613d 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1002,6 +1002,8 @@ static void readConfigs(opt::InputArgList &args) {
getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
config->thinLTOPrefixReplace =
getOldNewOptions(args, OPT_thinlto_prefix_replace_eq);
+ config->thinLTOModulesToCompile =
+ args::getStrings(args, OPT_thinlto_single_module_eq);
config->timeTraceEnabled = args.hasArg(OPT_time_trace);
config->timeTraceGranularity =
args::getInteger(args, OPT_time_trace_granularity, 500);
@@ -1973,7 +1975,10 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
// Likewise, --plugin-opt=emit-llvm and --plugin-opt=emit-asm are the
// options to create output files in bitcode or assembly code
// repsectively. No object files are generated.
- if (config->thinLTOIndexOnly || config->emitLLVM || config->ltoEmitAsm)
+ // Also bail out here when only certain thinLTO modules are specified for
+ // compilation. The intermediate object file are the expected output.
+ if (config->thinLTOIndexOnly || config->emitLLVM || config->ltoEmitAsm ||
+ !config->thinLTOModulesToCompile.empty())
return;
// Apply symbol renames for -wrap.
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 1f1c21764bc0..b8041afed6c9 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -140,6 +140,9 @@ static lto::Config createConfig() {
c.HasWholeProgramVisibility = config->ltoWholeProgramVisibility;
c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty();
+ for (const llvm::StringRef &name : config->thinLTOModulesToCompile)
+ c.ThinLTOModulesToCompile.emplace_back(name);
+
c.TimeTraceEnabled = config->timeTraceEnabled;
c.TimeTraceGranularity = config->timeTraceGranularity;
@@ -296,12 +299,14 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
},
cache));
- // Emit empty index files for non-indexed files
- for (StringRef s : thinIndices) {
- std::string path = getThinLTOOutputFile(s);
- openFile(path + ".thinlto.bc");
- if (config->thinLTOEmitImportsFiles)
- openFile(path + ".imports");
+ // Emit empty index files for non-indexed files but not in single-module mode.
+ if (config->thinLTOModulesToCompile.empty()) {
+ for (StringRef s : thinIndices) {
+ std::string path = getThinLTOOutputFile(s);
+ openFile(path + ".thinlto.bc");
+ if (config->thinLTOEmitImportsFiles)
+ openFile(path + ".imports");
+ }
}
if (config->thinLTOIndexOnly) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 939af8f510a7..0a16faa2b8fe 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -562,6 +562,8 @@ def thinlto_jobs: JJ<"thinlto-jobs=">,
HelpText<"Number of ThinLTO jobs. Default to --threads=">;
def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
+def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
+ HelpText<"Specific a single module to compile in ThinLTO mode, for debugging only">;
def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
def: F<"plugin-opt=debug-pass-manager">,
diff --git a/lld/test/ELF/lto/thinlto-single-module.ll b/lld/test/ELF/lto/thinlto-single-module.ll
new file mode 100644
index 000000000000..053021319351
--- /dev/null
+++ b/lld/test/ELF/lto/thinlto-single-module.ll
@@ -0,0 +1,69 @@
+; REQUIRES: x86
+; RUN: rm -fr %t && mkdir %t && cd %t
+; RUN: opt -thinlto-bc -o main.o %s
+; RUN: opt -thinlto-bc -o thin1.o %S/Inputs/thin1.ll
+; RUN: opt -thinlto-bc -o thin2.o %S/Inputs/thin2.ll
+; RUN: llvm-ar qcT thin.a thin1.o thin2.o
+
+;; --thinlto-single-module=main.o should result in only main.o compiled, of which
+;; the object code is saved in single1.o1. Note that single1.o is always the dummy
+;; output, aka ld-temp.o. There should be no more object files generated.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --lto-obj-path=single1.o
+; RUN: llvm-readelf -S -s single1.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single1.o1 | FileCheck %s --check-prefix=MAIN
+; RUN: not ls single1.o2
+; RUN: not ls a.out
+
+; DEFAULT: Value Size Type Bind Vis Ndx Name
+; DEFAULT: 0000000000000000 0 FILE LOCAL DEFAULT ABS ld-temp.o
+; MAIN: Value Size Type Bind Vis Ndx Name
+; MAIN: 0000000000000000 0 FILE LOCAL DEFAULT ABS thinlto-single-module.ll
+; MAIN-NEXT: 0000000000000000 3 FUNC GLOBAL DEFAULT 3 _start
+
+;; --thinlto-single-module=thin.a should result in only thin1.o and thin2.o compiled.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=thin.a --lto-obj-path=single2.o
+; RUN: llvm-readelf -S -s single2.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single2.o1 | FileCheck %s --check-prefix=FOO
+; RUN: llvm-readelf -S -s single2.o2 | FileCheck %s --check-prefix=BLAH
+; RUN: not ls single1.o3
+
+;; Multiple --thinlto-single-module uses should result in a combination of inputs compiled.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-single-module=thin2.o --lto-obj-path=single4.o
+; RUN: llvm-readelf -S -s single4.o | FileCheck %s --check-prefix=DEFAULT
+; RUN: llvm-readelf -S -s single4.o1 | FileCheck %s --check-prefix=MAIN
+; RUN: llvm-readelf -S -s single4.o2 | FileCheck %s --check-prefix=BLAH
+; RUN: not ls single4.o3
+
+; FOO: Value Size Type Bind Vis Ndx Name
+; FOO: 0000000000000000 0 FILE LOCAL DEFAULT ABS thin1.ll
+; FOO-NEXT: 0000000000000000 6 FUNC GLOBAL DEFAULT 3 foo
+; BLAH: Value Size Type Bind Vis Ndx Name
+; BLAH: 0000000000000000 0 FILE LOCAL DEFAULT ABS thin2.ll
+; BLAH-NEXT: 0000000000000000 4 FUNC GLOBAL DEFAULT 3 blah
+
+;; Check only main.o is in the result thin index file.
+;; Also check a *.thinlto.bc file generated for main.o only.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-index-only=single5.idx
+; RUN: ls main.o.thinlto.bc
+; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null
+; RUN: FileCheck %s --check-prefix=IDX < single5.idx
+; RUN: count 1 < single5.idx
+
+; IDX: main.o
+
+;; Check temporary output generated for main.o only.
+; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --save-temps
+; RUN: ls main.o.0.preopt.bc
+; RUN: not ls thin.*.0.preopt.bc
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-scei-ps4"
+
+declare i32 @blah(i32 %meh)
+declare i32 @foo(i32 %goo)
+
+define i32 @_start() {
+ call i32 @foo(i32 0)
+ call i32 @blah(i32 0)
+ ret i32 0
+}
diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h
index 20fed3488f29..0a3e52316460 100644
--- a/llvm/include/llvm/LTO/Config.h
+++ b/llvm/include/llvm/LTO/Config.h
@@ -130,6 +130,9 @@ struct Config {
/// Statistics output file path.
std::string StatsFile;
+ /// Specific thinLTO modules to compile.
+ std::vector<std::string> ThinLTOModulesToCompile;
+
/// Time trace enabled.
bool TimeTraceEnabled = false;
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index c25aa077304b..93456c0ae7ae 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringMap.h"
+#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/LTO/Config.h"
#include "llvm/Object/IRSymtab.h"
@@ -26,7 +27,6 @@
namespace llvm {
-class BitcodeModule;
class Error;
class IRMover;
class LLVMContext;
@@ -330,12 +330,17 @@ class LTO {
bool EmptyCombinedModule = true;
} RegularLTO;
+ using ModuleMapType = MapVector<StringRef, BitcodeModule>;
+
struct ThinLTOState {
ThinLTOState(ThinBackend Backend);
ThinBackend Backend;
ModuleSummaryIndex CombinedIndex;
- MapVector<StringRef, BitcodeModule> ModuleMap;
+ // The full set of bitcode modules in input order.
+ ModuleMapType ModuleMap;
+ // The bitcode modules to compile, if specified by the LTO Config.
+ Optional<ModuleMapType> ModulesToCompile;
DenseMap<GlobalValue::GUID, StringRef> PrevailingModuleForGUID;
} ThinLTO;
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a6c61c4b0892..f4cf0d23245d 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -870,12 +870,28 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
"Expected at most one ThinLTO module per bitcode file",
inconvertibleErrorCode());
+ if (!Conf.ThinLTOModulesToCompile.empty()) {
+ if (!ThinLTO.ModulesToCompile)
+ ThinLTO.ModulesToCompile = ModuleMapType();
+ // This is a fuzzy name matching where only modules with name containing the
+ // specified switch values are going to be compiled.
+ for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
+ if (BM.getModuleIdentifier().contains(Name)) {
+ ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
+ llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
+ << " to compile\n";
+ }
+ }
+ }
+
return Error::success();
}
unsigned LTO::getMaxTasks() const {
CalledGetMaxTasks = true;
- return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
+ auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
+ : ThinLTO.ModuleMap.size();
+ return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
}
// If only some of the modules were split, we cannot correctly handle
@@ -1312,6 +1328,11 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
if (ThinLTO.ModuleMap.empty())
return Error::success();
+ if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
+ llvm::errs() << "warning: [ThinLTO] No module compiled\n";
+ return Error::success();
+ }
+
if (Conf.CombinedIndexHook &&
!Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
return Error::success();
@@ -1416,10 +1437,13 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
AddStream, Cache);
+ auto &ModuleMap =
+ ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
+
// Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for combined
// module and parallel code generation partitions.
unsigned Task = RegularLTO.ParallelCodeGenParallelismLevel;
- for (auto &Mod : ThinLTO.ModuleMap) {
+ for (auto &Mod : ModuleMap) {
if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
ExportLists[Mod.first],
ResolvedODR[Mod.first], ThinLTO.ModuleMap))
More information about the llvm-commits
mailing list