[lld] r246342 - COFF: Implement parallel LTO code generation.
Peter Collingbourne via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 15:16:09 PDT 2015
Author: pcc
Date: Fri Aug 28 17:16:09 2015
New Revision: 246342
URL: http://llvm.org/viewvc/llvm-project?rev=246342&view=rev
Log:
COFF: Implement parallel LTO code generation.
This is exposed via a new flag /opt:lldltojobs=N, where N is the number of
code generation threads.
Differential Revision: http://reviews.llvm.org/D12309
Added:
lld/trunk/test/COFF/lto-parallel.ll
Modified:
lld/trunk/COFF/Config.h
lld/trunk/COFF/Driver.cpp
lld/trunk/COFF/SymbolTable.cpp
lld/trunk/COFF/SymbolTable.h
lld/trunk/test/COFF/weak-external.test
lld/trunk/test/COFF/weak-external3.test
Modified: lld/trunk/COFF/Config.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/Config.h?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/COFF/Config.h (original)
+++ lld/trunk/COFF/Config.h Fri Aug 28 17:16:09 2015
@@ -91,6 +91,9 @@ struct Configuration {
// Used for /opt:lldlto=N
unsigned LTOOptLevel = 2;
+ // Used for /opt:lldltojobs=N
+ unsigned LTOJobs = 1;
+
// Used for /merge:from=to (e.g. /merge:.rdata=.text)
std::map<StringRef, StringRef> Merge;
Modified: lld/trunk/COFF/Driver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/Driver.cpp?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/COFF/Driver.cpp (original)
+++ lld/trunk/COFF/Driver.cpp Fri Aug 28 17:16:09 2015
@@ -372,9 +372,14 @@ void LinkerDriver::link(llvm::ArrayRef<c
if (StringRef(S).startswith("lldlto=")) {
StringRef OptLevel = StringRef(S).substr(7);
if (OptLevel.getAsInteger(10, Config->LTOOptLevel) ||
- Config->LTOOptLevel > 3) {
+ Config->LTOOptLevel > 3)
error("/opt:lldlto: invalid optimization level: " + OptLevel);
- }
+ continue;
+ }
+ if (StringRef(S).startswith("lldltojobs=")) {
+ StringRef Jobs = StringRef(S).substr(11);
+ if (Jobs.getAsInteger(10, Config->LTOJobs) || Config->LTOJobs == 0)
+ error("/opt:lldltojobs: invalid job count: " + Jobs);
continue;
}
if (S != "ref" && S != "icf" && S != "noicf" &&
@@ -594,9 +599,9 @@ void LinkerDriver::link(llvm::ArrayRef<c
Symtab.run();
}
- // Do LTO by compiling bitcode input files to a native COFF file
- // then link that file.
- Symtab.addCombinedLTOObject();
+ // Do LTO by compiling bitcode input files to a set of native COFF files then
+ // link those files.
+ Symtab.addCombinedLTOObjects();
// Make sure we have resolved all symbols.
Symtab.reportRemainingUndefines(/*Resolve=*/true);
Modified: lld/trunk/COFF/SymbolTable.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/SymbolTable.cpp?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/COFF/SymbolTable.cpp (original)
+++ lld/trunk/COFF/SymbolTable.cpp Fri Aug 28 17:16:09 2015
@@ -331,21 +331,7 @@ void SymbolTable::printMap(llvm::raw_ost
}
}
-void SymbolTable::addCombinedLTOObject() {
- if (BitcodeFiles.empty())
- return;
-
- // Diagnose any undefined symbols early, but do not resolve weak externals,
- // as resolution breaks the invariant that each Symbol points to a unique
- // SymbolBody, which we rely on to replace DefinedBitcode symbols correctly.
- reportRemainingUndefines(/*Resolve=*/false);
-
- // Create an object file and add it to the symbol table by replacing any
- // DefinedBitcode symbols with the definitions in the object file.
- LTOCodeGenerator CG;
- CG.setOptLevel(Config->LTOOptLevel);
- ObjectFile *Obj = createLTOObject(&CG);
-
+void SymbolTable::addCombinedLTOObject(ObjectFile *Obj) {
for (SymbolBody *Body : Obj->getSymbols()) {
if (!Body->isExternal())
continue;
@@ -371,6 +357,25 @@ void SymbolTable::addCombinedLTOObject()
if (Comp < 0)
Sym->Body = Body;
}
+}
+
+void SymbolTable::addCombinedLTOObjects() {
+ if (BitcodeFiles.empty())
+ return;
+
+ // Diagnose any undefined symbols early, but do not resolve weak externals,
+ // as resolution breaks the invariant that each Symbol points to a unique
+ // SymbolBody, which we rely on to replace DefinedBitcode symbols correctly.
+ reportRemainingUndefines(/*Resolve=*/false);
+
+ // Create an object file and add it to the symbol table by replacing any
+ // DefinedBitcode symbols with the definitions in the object file.
+ LTOCodeGenerator CG;
+ CG.setOptLevel(Config->LTOOptLevel);
+ std::vector<ObjectFile *> Objs = createLTOObjects(&CG);
+
+ for (ObjectFile *Obj : Objs)
+ addCombinedLTOObject(Obj);
size_t NumBitcodeFiles = BitcodeFiles.size();
run();
@@ -379,8 +384,8 @@ void SymbolTable::addCombinedLTOObject()
}
// Combine and compile bitcode files and then return the result
-// as a regular COFF object file.
-ObjectFile *SymbolTable::createLTOObject(LTOCodeGenerator *CG) {
+// as a vector of regular COFF object files.
+std::vector<ObjectFile *> SymbolTable::createLTOObjects(LTOCodeGenerator *CG) {
// All symbols referenced by non-bitcode objects must be preserved.
for (ObjectFile *File : ObjectFiles)
for (SymbolBody *Body : File->getSymbols())
@@ -406,14 +411,32 @@ ObjectFile *SymbolTable::createLTOObject
CG->addModule(BitcodeFiles[I]->getModule());
std::string ErrMsg;
- LTOMB = CG->compile(false, false, false, ErrMsg); // take MB ownership
- if (!LTOMB)
+ if (!CG->optimize(false, false, false, ErrMsg))
+ error(ErrMsg);
+
+ Objs.resize(Config->LTOJobs);
+ // Use std::list to avoid invalidation of pointers in OSPtrs.
+ std::list<raw_svector_ostream> OSs;
+ std::vector<raw_pwrite_stream *> OSPtrs;
+ for (SmallVector<char, 0> &Obj : Objs) {
+ OSs.emplace_back(Obj);
+ OSPtrs.push_back(&OSs.back());
+ }
+
+ if (!CG->compileOptimized(OSPtrs, ErrMsg))
error(ErrMsg);
- auto *Obj = new ObjectFile(LTOMB->getMemBufferRef());
- Files.emplace_back(Obj);
- ObjectFiles.push_back(Obj);
- Obj->parse();
- return Obj;
+
+ std::vector<ObjectFile *> ObjFiles;
+ for (SmallVector<char, 0> &Obj : Objs) {
+ auto *ObjFile = new ObjectFile(
+ MemoryBufferRef(StringRef(Obj.data(), Obj.size()), "<LTO object>"));
+ Files.emplace_back(ObjFile);
+ ObjectFiles.push_back(ObjFile);
+ ObjFile->parse();
+ ObjFiles.push_back(ObjFile);
+ }
+
+ return ObjFiles;
}
} // namespace coff
Modified: lld/trunk/COFF/SymbolTable.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/COFF/SymbolTable.h?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/COFF/SymbolTable.h (original)
+++ lld/trunk/COFF/SymbolTable.h Fri Aug 28 17:16:09 2015
@@ -68,10 +68,10 @@ public:
// Print a layout map to OS.
void printMap(llvm::raw_ostream &OS);
- // Build a COFF object representing the combined contents of BitcodeFiles
- // and add it to the symbol table. Called after all files are added and
- // before the writer writes results to a file.
- void addCombinedLTOObject();
+ // Build a set of COFF objects representing the combined contents of
+ // BitcodeFiles and add them to the symbol table. Called after all files are
+ // added and before the writer writes results to a file.
+ void addCombinedLTOObjects();
// The writer needs to handle DLL import libraries specially in
// order to create the import descriptor table.
@@ -98,7 +98,8 @@ private:
StringRef findByPrefix(StringRef Prefix);
void addMemberFile(Lazy *Body);
- ObjectFile *createLTOObject(llvm::LTOCodeGenerator *CG);
+ void addCombinedLTOObject(ObjectFile *Obj);
+ std::vector<ObjectFile *> createLTOObjects(llvm::LTOCodeGenerator *CG);
llvm::DenseMap<StringRef, Symbol *> Symtab;
@@ -107,7 +108,7 @@ private:
std::vector<InputFile *> ObjectQueue;
std::vector<BitcodeFile *> BitcodeFiles;
- std::unique_ptr<MemoryBuffer> LTOMB;
+ std::vector<SmallVector<char, 0>> Objs;
llvm::BumpPtrAllocator Alloc;
};
Added: lld/trunk/test/COFF/lto-parallel.ll
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/lto-parallel.ll?rev=246342&view=auto
==============================================================================
--- lld/trunk/test/COFF/lto-parallel.ll (added)
+++ lld/trunk/test/COFF/lto-parallel.ll Fri Aug 28 17:16:09 2015
@@ -0,0 +1,20 @@
+; RUN: llvm-as -o %t.obj %s
+; RUN: lld-link /out:%t.exe /entry:foo /include:bar /opt:lldltojobs=2 /subsystem:console /lldmap:%t.map %t.obj
+; RUN: FileCheck %s < %t.map
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: <lto object>:
+; CHECK: foo
+define void @foo() {
+ call void @bar()
+ ret void
+}
+
+; CHECK: <lto object>:
+; CHECK: bar
+define void @bar() {
+ call void @foo()
+ ret void
+}
Modified: lld/trunk/test/COFF/weak-external.test
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/weak-external.test?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/test/COFF/weak-external.test (original)
+++ lld/trunk/test/COFF/weak-external.test Fri Aug 28 17:16:09 2015
@@ -4,7 +4,7 @@
# RUN: lld-link /out:%t2.exe /entry:g /subsystem:console /lldmap:%t2.map %t.obj %t.lto.obj
# RUN: FileCheck %s < %t2.map
-# CHECK: lto-llvm{{.*}}:
+# CHECK: <lto object>:
# CHECK-NOT: :
# CHECK: {{ g$}}
Modified: lld/trunk/test/COFF/weak-external3.test
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/COFF/weak-external3.test?rev=246342&r1=246341&r2=246342&view=diff
==============================================================================
--- lld/trunk/test/COFF/weak-external3.test (original)
+++ lld/trunk/test/COFF/weak-external3.test Fri Aug 28 17:16:09 2015
@@ -5,7 +5,7 @@
# RUN: lld-link /out:%t2.exe /entry:f /subsystem:console /lldmap:%t2.map %t.obj %t.lto.obj
# RUN: FileCheck --check-prefix=CHECK2 %s < %t2.map
-# CHECK1: lto-llvm{{.*}}:
+# CHECK1: <lto object>:
# CHECK1-NOT: :
# CHECK1: {{ g$}}
More information about the llvm-commits
mailing list