[llvm] r269067 - [ThinLTO] Add option to emit imports files for distributed backends
Teresa Johnson via llvm-commits
llvm-commits at lists.llvm.org
Tue May 10 08:54:09 PDT 2016
Author: tejohnson
Date: Tue May 10 10:54:09 2016
New Revision: 269067
URL: http://llvm.org/viewvc/llvm-project?rev=269067&view=rev
Log:
[ThinLTO] Add option to emit imports files for distributed backends
Summary:
Add support for emission of plaintext lists of the imported files for
each distributed backend compilation. Used for distributed build file
staging.
Invoked with new gold-plugin thinlto-emit-imports-files option, which is
only valid with thinlto-index-only (i.e. for distributed builds), or
from llvm-lto with new -thinlto-action=emitimports value.
Depends on D19556.
Reviewers: joker.eph
Subscribers: llvm-commits, joker.eph
Differential Revision: http://reviews.llvm.org/D19636
Added:
llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll
llvm/trunk/test/ThinLTO/X86/emit_imports.ll
llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll
Modified:
llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h
llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h
llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp
llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp
llvm/trunk/tools/gold/gold-plugin.cpp
llvm/trunk/tools/llvm-lto/llvm-lto.cpp
Modified: llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h (original)
+++ llvm/trunk/include/llvm/LTO/ThinLTOCodeGenerator.h Tue May 10 10:54:09 2016
@@ -202,6 +202,12 @@ public:
void promote(Module &Module, ModuleSummaryIndex &Index);
/**
+ * Compute and emit the imported files for module at \p ModulePath.
+ */
+ static void emitImports(StringRef ModulePath, StringRef OutputName,
+ ModuleSummaryIndex &Index);
+
+ /**
* Perform cross-module importing for the module identified by
* ModuleIdentifier.
*/
Modified: llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h (original)
+++ llvm/trunk/include/llvm/Transforms/IPO/FunctionImport.h Tue May 10 10:54:09 2016
@@ -103,6 +103,10 @@ void gatherImportedSummariesForModule(
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
const StringMap<FunctionImporter::ImportMapTy> &ImportLists,
std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
+
+std::error_code
+EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename,
+ const StringMap<FunctionImporter::ImportMapTy> &ImportLists);
}
#endif // LLVM_FUNCTIONIMPORT_H
Modified: llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp (original)
+++ llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp Tue May 10 10:54:09 2016
@@ -743,6 +743,30 @@ void ThinLTOCodeGenerator::gatherImporte
}
/**
+ * Emit the list of files needed for importing into module.
+ */
+void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
+ StringRef OutputName,
+ ModuleSummaryIndex &Index) {
+ auto ModuleCount = Index.modulePaths().size();
+
+ // Collect for each module the list of function it defines (GUID -> Summary).
+ StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
+ Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
+
+ // Generate import/export list
+ StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
+ StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
+ ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
+ ExportLists);
+
+ std::error_code EC;
+ if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists)))
+ report_fatal_error(Twine("Failed to open ") + OutputName +
+ " to save imports lists\n");
+}
+
+/**
* Perform internalization.
*/
void ThinLTOCodeGenerator::internalize(Module &TheModule,
Modified: llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp Tue May 10 10:54:09 2016
@@ -445,6 +445,21 @@ void llvm::gatherImportedSummariesForMod
}
}
+/// Emit the files \p ModulePath will import from into \p OutputFilename.
+std::error_code llvm::EmitImportsFiles(
+ StringRef ModulePath, StringRef OutputFilename,
+ const StringMap<FunctionImporter::ImportMapTy> &ImportLists) {
+ auto ModuleImports = ImportLists.find(ModulePath);
+ std::error_code EC;
+ raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
+ if (EC)
+ return EC;
+ if (ModuleImports != ImportLists.end())
+ for (auto &ILI : ModuleImports->second)
+ ImportsOS << ILI.first() << "\n";
+ return std::error_code();
+}
+
// Automatically import functions in Module \p DestModule based on the summaries
// index.
//
Added: llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll?rev=269067&view=auto
==============================================================================
--- llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll (added)
+++ llvm/trunk/test/ThinLTO/X86/Inputs/emit_imports.ll Tue May 10 10:54:09 2016
@@ -0,0 +1,4 @@
+define void @g() {
+entry:
+ ret void
+}
Added: llvm/trunk/test/ThinLTO/X86/emit_imports.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/ThinLTO/X86/emit_imports.ll?rev=269067&view=auto
==============================================================================
--- llvm/trunk/test/ThinLTO/X86/emit_imports.ll (added)
+++ llvm/trunk/test/ThinLTO/X86/emit_imports.ll Tue May 10 10:54:09 2016
@@ -0,0 +1,21 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/emit_imports.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc
+; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc
+
+; The imports file for this module contains the bitcode file for
+; Inputs/emit_imports.ll
+; RUN: cat %t1.bc.imports | count 1
+; RUN: cat %t1.bc.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: emit_imports.ll.tmp2.bc
+
+; The imports file for Input/emit_imports.ll is empty as it does not import anything.
+; RUN: cat %t2.bc.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+ call void (...) @g()
+ ret void
+}
Added: llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll?rev=269067&view=auto
==============================================================================
--- llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll (added)
+++ llvm/trunk/test/tools/gold/X86/thinlto_emit_imports.ll Tue May 10 10:54:09 2016
@@ -0,0 +1,27 @@
+; Generate summary sections and test gold handling.
+; RUN: opt -module-summary %s -o %t.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
+
+; Ensure gold generates imports files if requested for distributed backends.
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN: --plugin-opt=thinlto \
+; RUN: --plugin-opt=thinlto-index-only \
+; RUN: --plugin-opt=thinlto-emit-imports-files \
+; RUN: -shared %t.o %t2.o -o %t3
+
+; The imports file for this module contains the bitcode file for
+; Inputs/thinlto.ll
+; RUN: cat %t.o.imports | count 1
+; RUN: cat %t.o.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: test/tools/gold/X86/Output/thinlto_emit_imports.ll.tmp2.o
+
+; The imports file for Input/thinlto.ll is empty as it does not import anything.
+; RUN: cat %t2.o.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+ call void (...) @g()
+ ret void
+}
Modified: llvm/trunk/tools/gold/gold-plugin.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)
+++ llvm/trunk/tools/gold/gold-plugin.cpp Tue May 10 10:54:09 2016
@@ -184,6 +184,11 @@ namespace options {
// the import decisions, and exit afterwards. The assumption is
// that the build system will launch the backend processes.
static bool thinlto_index_only = false;
+ // If true, when generating individual index files for distributed backends,
+ // also generate a "${bitcodefile}.imports" file at the same location for each
+ // bitcode file, listing the files it imports from in plain text. This is to
+ // support distributed build file staging.
+ static bool thinlto_emit_imports_files = false;
// Additional options to pass into the code generator.
// Note: This array will contain all plugin options which are not claimed
// as plugin exclusive to pass to the code generator.
@@ -217,6 +222,8 @@ namespace options {
thinlto = true;
} else if (opt == "thinlto-index-only") {
thinlto_index_only = true;
+ } else if (opt == "thinlto-emit-imports-files") {
+ thinlto_emit_imports_files = true;
} else if (opt.size() == 2 && opt[0] == 'O') {
if (opt[1] < '0' || opt[1] > '3')
message(LDPL_FATAL, "Optimization level must be between 0 and 3");
@@ -1209,6 +1216,10 @@ static ld_plugin_status thinLTOLink(raw_
CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
}
+ if (options::thinlto_emit_imports_files && !options::thinlto_index_only)
+ message(LDPL_WARNING,
+ "thinlto-emit-imports-files ignored unless thinlto-index-only");
+
if (options::thinlto_index_only) {
// Collect for each module the list of function it defines (GUID ->
// Summary).
@@ -1244,6 +1255,15 @@ static ld_plugin_status thinLTOLink(raw_
ModuleToDefinedGVSummaries, ImportLists,
ModuleToSummariesForIndex);
WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
+
+ if (options::thinlto_emit_imports_files) {
+ if ((EC = EmitImportsFiles(
+ InputFile.file().name,
+ (Twine(InputFile.file().name) + ".imports").str(),
+ ImportLists)))
+ message(LDPL_FATAL, "Unable to open %s.imports",
+ InputFile.file().name, EC.message().c_str());
+ }
}
cleanup_hook();
Modified: llvm/trunk/tools/llvm-lto/llvm-lto.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-lto/llvm-lto.cpp?rev=269067&r1=269066&r2=269067&view=diff
==============================================================================
--- llvm/trunk/tools/llvm-lto/llvm-lto.cpp (original)
+++ llvm/trunk/tools/llvm-lto/llvm-lto.cpp Tue May 10 10:54:09 2016
@@ -67,6 +67,7 @@ static cl::opt<bool>
enum ThinLTOModes {
THINLINK,
THINDISTRIBUTE,
+ THINEMITIMPORTS,
THINPROMOTE,
THINIMPORT,
THININTERNALIZE,
@@ -83,6 +84,8 @@ cl::opt<ThinLTOModes> ThinLTOMode(
"ThinLink: produces the index by linking only the summaries."),
clEnumValN(THINDISTRIBUTE, "distributedindexes",
"Produces individual indexes for distributed backends."),
+ clEnumValN(THINEMITIMPORTS, "emitimports",
+ "Emit imports files for distributed backends."),
clEnumValN(THINPROMOTE, "promote",
"Perform pre-import promotion (requires -thinlto-index)."),
clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -359,6 +362,8 @@ public:
return thinLink();
case THINDISTRIBUTE:
return distributedIndexes();
+ case THINEMITIMPORTS:
+ return emitImports();
case THINPROMOTE:
return promote();
case THINIMPORT:
@@ -431,6 +436,25 @@ private:
}
}
+ /// Load the combined index from disk, compute the imports, and emit
+ /// the import file lists for each module to disk.
+ void emitImports() {
+ if (InputFilenames.size() != 1 && !OutputFilename.empty())
+ report_fatal_error("Can't handle a single output filename and multiple "
+ "input files, do not provide an output filename and "
+ "the output files will be suffixed from the input "
+ "ones.");
+
+ auto Index = loadCombinedIndex();
+ for (auto &Filename : InputFilenames) {
+ std::string OutputName = OutputFilename;
+ if (OutputName.empty()) {
+ OutputName = Filename + ".imports";
+ }
+ ThinLTOCodeGenerator::emitImports(Filename, OutputName, *Index);
+ }
+ }
+
/// Load the combined index from disk, then load every file referenced by
/// the index and add them to the generator, finally perform the promotion
/// on the files mentioned on the command line (these must match the index
More information about the llvm-commits
mailing list