[llvm] r282100 - [ThinLTO] Emit files for distributed builds for all modules
Teresa Johnson via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 21 12:12:05 PDT 2016
Author: tejohnson
Date: Wed Sep 21 14:12:05 2016
New Revision: 282100
URL: http://llvm.org/viewvc/llvm-project?rev=282100&view=rev
Log:
[ThinLTO] Emit files for distributed builds for all modules
With the new LTO API in r278338, we stopped emitting the individual
index files and imports files for some modules in the distributed backend
case (thinlto-index-only plugin option).
Specifically, this is when the linker decides not to include a module in the
link, because it was in an archive library and did not have a strong
reference to it. Not creating the expected output files makes the
distributed build system implementation more difficult, in terms of
checking for the expected outputs of the thin link, and scheduling the
backend jobs. To address this, the gold-plugin will write dummy empty
.thinlto.bc and .imports files for modules not included in the link
(which LTO never sees).
Augmented a gold v1.12+ test, since that version of gold has the handling
for notifying on modules not being included in the link.
Modified:
llvm/trunk/include/llvm/LTO/LTO.h
llvm/trunk/lib/LTO/LTO.cpp
llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
llvm/trunk/tools/gold/gold-plugin.cpp
Modified: llvm/trunk/include/llvm/LTO/LTO.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTO.h?rev=282100&r1=282099&r2=282100&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/LTO.h (original)
+++ llvm/trunk/include/llvm/LTO/LTO.h Wed Sep 21 14:12:05 2016
@@ -85,6 +85,13 @@ void thinLTOInternalizeAndPromoteInIndex
namespace lto {
+/// Given the original \p Path to an output file, replace any path
+/// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
+/// resulting directory if it does not yet exist.
+std::string getThinLTOOutputFile(const std::string &Path,
+ const std::string &OldPrefix,
+ const std::string &NewPrefix);
+
class LTO;
struct SymbolResolution;
class ThinBackendProc;
Modified: llvm/trunk/lib/LTO/LTO.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTO.cpp?rev=282100&r1=282099&r2=282100&view=diff
==============================================================================
--- llvm/trunk/lib/LTO/LTO.cpp (original)
+++ llvm/trunk/lib/LTO/LTO.cpp Wed Sep 21 14:12:05 2016
@@ -609,6 +609,26 @@ ThinBackend lto::createInProcessThinBack
};
}
+// Given the original \p Path to an output file, replace any path
+// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
+// resulting directory if it does not yet exist.
+std::string lto::getThinLTOOutputFile(const std::string &Path,
+ const std::string &OldPrefix,
+ const std::string &NewPrefix) {
+ if (OldPrefix.empty() && NewPrefix.empty())
+ return Path;
+ SmallString<128> NewPath(Path);
+ llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
+ StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
+ if (!ParentPath.empty()) {
+ // Make sure the new directory exists, creating it if necessary.
+ if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
+ llvm::errs() << "warning: could not create directory '" << ParentPath
+ << "': " << EC.message() << '\n';
+ }
+ return NewPath.str();
+}
+
class WriteIndexesThinBackend : public ThinBackendProc {
std::string OldPrefix, NewPrefix;
bool ShouldEmitImportsFiles;
@@ -627,26 +647,6 @@ public:
ShouldEmitImportsFiles(ShouldEmitImportsFiles),
LinkedObjectsFileName(LinkedObjectsFileName) {}
- /// Given the original \p Path to an output file, replace any path
- /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
- /// resulting directory if it does not yet exist.
- std::string getThinLTOOutputFile(const std::string &Path,
- const std::string &OldPrefix,
- const std::string &NewPrefix) {
- if (OldPrefix.empty() && NewPrefix.empty())
- return Path;
- SmallString<128> NewPath(Path);
- llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
- StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
- if (!ParentPath.empty()) {
- // Make sure the new directory exists, creating it if necessary.
- if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
- llvm::errs() << "warning: could not create directory '" << ParentPath
- << "': " << EC.message() << '\n';
- }
- return NewPath.str();
- }
-
Error start(
unsigned Task, MemoryBufferRef MBRef,
const FunctionImporter::ImportMapTy &ImportList,
Modified: llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll?rev=282100&r1=282099&r2=282100&view=diff
==============================================================================
--- llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll (original)
+++ llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll Wed Sep 21 14:12:05 2016
@@ -9,14 +9,26 @@
; be included in the link, and not %t2.o since it is within
; a library (--start-lib/--end-lib pair) and not strongly referenced.
; Note that the support for detecting this is in gold v1.12.
+; RUN: rm -f %t.o.thinlto.bc
+; RUN: rm -f %t2.o.thinlto.bc
+; RUN: rm -f %t.o.imports
+; RUN: rm -f %t2.o.imports
; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
; RUN: --plugin-opt=thinlto \
; RUN: --plugin-opt=thinlto-index-only=%t3 \
+; RUN: --plugin-opt=thinlto-emit-imports-files \
; RUN: -m elf_x86_64 \
; RUN: -o %t4 \
; RUN: %t.o \
; RUN: --start-lib %t2.o --end-lib
+; Ensure that the expected output files are created, even for the file
+; the linker decided not to include in the link.
+; RUN: ls %t.o.thinlto.bc
+; RUN: ls %t2.o.thinlto.bc
+; RUN: ls %t.o.imports
+; RUN: ls %t2.o.imports
+
; RUN: cat %t3 | FileCheck %s
; CHECK: thinlto_emit_linked_objects.ll.tmp.o
; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o
Modified: llvm/trunk/tools/gold/gold-plugin.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=282100&r1=282099&r2=282100&view=diff
==============================================================================
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)
+++ llvm/trunk/tools/gold/gold-plugin.cpp Wed Sep 21 14:12:05 2016
@@ -752,6 +752,34 @@ static std::unique_ptr<LTO> createLTO()
ParallelCodeGenParallelismLevel);
}
+// Write empty files that may be expected by a distributed build
+// system when invoked with thinlto_index_only. This is invoked when
+// the linker has decided not to include the given module in the
+// final link. Frequently the distributed build system will want to
+// confirm that all expected outputs are created based on all of the
+// modules provided to the linker.
+static void writeEmptyDistributedBuildOutputs(std::string &ModulePath,
+ std::string &OldPrefix,
+ std::string &NewPrefix) {
+ std::string NewModulePath =
+ getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
+ std::error_code EC;
+ {
+ raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
+ sys::fs::OpenFlags::F_None);
+ if (EC)
+ message(LDPL_FATAL, "Failed to write '%s': %s",
+ (NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
+ }
+ if (options::thinlto_emit_imports_files) {
+ raw_fd_ostream OS(NewModulePath + ".imports", EC,
+ sys::fs::OpenFlags::F_None);
+ if (EC)
+ message(LDPL_FATAL, "Failed to write '%s': %s",
+ (NewModulePath + ".imports").c_str(), EC.message().c_str());
+ }
+}
+
/// gold informs us that all symbols have been read. At this point, we use
/// get_symbols to see if any of our definitions have been overridden by a
/// native object file. Then, perform optimization and codegen.
@@ -771,13 +799,22 @@ static ld_plugin_status allSymbolsReadHo
std::unique_ptr<LTO> Lto = createLTO();
+ std::string OldPrefix, NewPrefix;
+ if (options::thinlto_index_only)
+ getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
+
for (claimed_file &F : Modules) {
if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
HandleToInputFile.insert(std::make_pair(
F.leader_handle, llvm::make_unique<PluginInputFile>(F.handle)));
const void *View = getSymbolsAndView(F);
- if (!View)
+ if (!View) {
+ if (options::thinlto_index_only)
+ // Write empty output files that may be expected by the distributed
+ // build system.
+ writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix);
continue;
+ }
addModule(*Lto, F, View);
}
More information about the llvm-commits
mailing list