[llvm] r325411 - [ThinLTO] Allow indexing to request backend to ignore the module
Vitaly Buka via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 16 15:38:22 PST 2018
Author: vitalybuka
Date: Fri Feb 16 15:38:22 2018
New Revision: 325411
URL: http://llvm.org/viewvc/llvm-project?rev=325411&view=rev
Log:
[ThinLTO] Allow indexing to request backend to ignore the module
Summary:
Gold plugin does not add pass to ThinLTO modules without useful symbols.
In this case ThinLTO can't create corresponding index file and some features, like CFI,
cannot be processes by backed correctly without index.
Given that we don't need the backed output we can request it to avoid
processing the module. This is implemented by this patch using new
"SkipModuleByDistributedBackend" flag.
Reviewers: pcc, tejohnson
Subscribers: mehdi_amini, inglorion, eraman, cfe-commits
Differential Revision: https://reviews.llvm.org/D42995
Modified:
llvm/trunk/include/llvm/IR/ModuleSummaryIndex.h
llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
llvm/trunk/tools/gold/gold-plugin.cpp
Modified: llvm/trunk/include/llvm/IR/ModuleSummaryIndex.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/ModuleSummaryIndex.h?rev=325411&r1=325410&r2=325411&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/ModuleSummaryIndex.h (original)
+++ llvm/trunk/include/llvm/IR/ModuleSummaryIndex.h Fri Feb 16 15:38:22 2018
@@ -682,6 +682,13 @@ private:
/// considered live.
bool WithGlobalValueDeadStripping = false;
+ /// Indicates that distributed backend should skip compilation of the
+ /// module. Flag is suppose to be set by distributed ThinLTO indexing
+ /// when it detected that the module is not needed during the final
+ /// linking. As result distributed backend should just output a minimal
+ /// valid object file.
+ bool SkipModuleByDistributedBackend = false;
+
/// If true then we're performing analysis of IR module, filling summary
/// accordingly. The value of 'false' means we're reading summary from
/// BC or YAML source. Affects the type of value stored in NameOrGV union
@@ -718,6 +725,13 @@ public:
WithGlobalValueDeadStripping = true;
}
+ bool skipModuleByDistributedBackend() const {
+ return SkipModuleByDistributedBackend;
+ }
+ void setSkipModuleByDistributedBackend() {
+ SkipModuleByDistributedBackend = true;
+ }
+
bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
return !WithGlobalValueDeadStripping || GVS->isLive();
}
Modified: llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp?rev=325411&r1=325410&r2=325411&view=diff
==============================================================================
--- llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp (original)
+++ llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp Fri Feb 16 15:38:22 2018
@@ -5186,11 +5186,14 @@ Error ModuleSummaryIndexBitcodeReader::p
case bitc::FS_FLAGS: { // [flags]
uint64_t Flags = Record[0];
// Scan flags (set only on the combined index).
- assert(Flags <= 1 && "Unexpected bits in flag");
+ assert(Flags <= 0x3 && "Unexpected bits in flag");
// 1 bit: WithGlobalValueDeadStripping flag.
if (Flags & 0x1)
TheIndex.setWithGlobalValueDeadStripping();
+ // 1 bit: SkipModuleByDistributedBackend flag.
+ if (Flags & 0x2)
+ TheIndex.setSkipModuleByDistributedBackend();
break;
}
case bitc::FS_VALUE_GUID: { // [valueid, refguid]
Modified: llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp?rev=325411&r1=325410&r2=325411&view=diff
==============================================================================
--- llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp (original)
+++ llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp Fri Feb 16 15:38:22 2018
@@ -3604,10 +3604,13 @@ void IndexBitcodeWriter::writeCombinedGl
Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
- // Write the index flags. Currently we only write a single flag, the value of
- // withGlobalValueDeadStripping, which only applies to the combined index.
- Stream.EmitRecord(bitc::FS_FLAGS,
- ArrayRef<uint64_t>{Index.withGlobalValueDeadStripping()});
+ // Write the index flags.
+ uint64_t Flags = 0;
+ if (Index.withGlobalValueDeadStripping())
+ Flags |= 0x1;
+ if (Index.skipModuleByDistributedBackend())
+ Flags |= 0x2;
+ Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
for (const auto &GVI : valueIds()) {
Stream.EmitRecord(bitc::FS_VALUE_GUID,
Modified: llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll?rev=325411&r1=325410&r2=325411&view=diff
==============================================================================
--- llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll (original)
+++ llvm/trunk/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll Fri Feb 16 15:38:22 2018
@@ -31,6 +31,24 @@
; RUN: ls %t2.o.imports
; RUN: ls %t3.o.imports
+; Regular *thinlto.bc file. "SkipModuleByDistributedBackend" flag (0x2)
+; should not be set.
+; RUN: llvm-bcanalyzer --dump %t1.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC1
+; CHECK-BC1: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC1: <FLAGS op0=1/>
+; CHECK-BC1: </GLOBALVAL_SUMMARY_BLOCK
+
+; Nothing interesting in the corresponding object file, so
+; "SkipModuleByDistributedBackend" flag (0x2) should be set.
+; RUN: llvm-bcanalyzer --dump %t2.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC2
+; CHECK-BC2: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC2: <FLAGS op0=2/>
+; CHECK-BC2: </GLOBALVAL_SUMMARY_BLOCK
+
+; Empty as the corresponding object file is not ThinTLO.
+; RUN: not llvm-bcanalyzer --dump %t3.o.thinlto.bc 2>&1 | FileCheck %s -check-prefixes=CHECK-BC3
+; CHECK-BC3: LLVM ERROR: Unexpected end of file
+
; RUN: cat %t.index | FileCheck %s
; CHECK: thinlto_emit_linked_objects.ll.tmp1.o
; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o
Modified: llvm/trunk/tools/gold/gold-plugin.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=325411&r1=325410&r2=325411&view=diff
==============================================================================
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)
+++ llvm/trunk/tools/gold/gold-plugin.cpp Fri Feb 16 15:38:22 2018
@@ -811,9 +811,14 @@ static std::unique_ptr<LTO> createLTO(In
// final link. Frequently the distributed build system will want to
// confirm that all expected outputs are created based on all of the
// modules provided to the linker.
+// If SkipModule is true then .thinlto.bc should contain just
+// SkipModuleByDistributedBackend flag which requests distributed backend
+// to skip the compilation of the corresponding module and produce an empty
+// object file.
static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
const std::string &OldPrefix,
- const std::string &NewPrefix) {
+ const std::string &NewPrefix,
+ bool SkipModule) {
std::string NewModulePath =
getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
std::error_code EC;
@@ -823,6 +828,12 @@ static void writeEmptyDistributedBuildOu
if (EC)
message(LDPL_FATAL, "Failed to write '%s': %s",
(NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
+
+ if (SkipModule) {
+ ModuleSummaryIndex Index(false);
+ Index.setSkipModuleByDistributedBackend();
+ WriteIndexToFile(Index, OS, nullptr);
+ }
}
if (options::thinlto_emit_imports_files) {
raw_fd_ostream OS(NewModulePath + ".imports", EC,
@@ -878,6 +889,11 @@ static ld_plugin_status allSymbolsReadHo
assert(ObjFilename.second);
if (const void *View = getSymbolsAndView(F))
addModule(*Lto, F, View, ObjFilename.first->first());
+ else if (options::thinlto_index_only) {
+ ObjFilename.first->second = true;
+ writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix,
+ /* SkipModule */ true);
+ }
}
SmallString<128> Filename;
@@ -895,7 +911,7 @@ static ld_plugin_status allSymbolsReadHo
auto AddStream =
[&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
IsTemporary[Task] = !SaveTemps;
- int FD = getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps,
+ int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Filenames[Task], Task);
return llvm::make_unique<lto::NativeObjectStream>(
llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
@@ -920,7 +936,7 @@ static ld_plugin_status allSymbolsReadHo
for (auto &Identifier : ObjectToIndexFileState)
if (!Identifier.getValue())
writeEmptyDistributedBuildOutputs(Identifier.getKey(), OldPrefix,
- NewPrefix);
+ NewPrefix, /* SkipModule */ false);
if (options::TheOutputType == options::OT_DISABLE ||
options::TheOutputType == options::OT_BC_ONLY)
More information about the llvm-commits
mailing list