[lld] r226281 - Run the resolver in parallel with the reader.
Rui Ueyama
ruiu at google.com
Fri Jan 16 07:54:14 PST 2015
Author: ruiu
Date: Fri Jan 16 09:54:13 2015
New Revision: 226281
URL: http://llvm.org/viewvc/llvm-project?rev=226281&view=rev
Log:
Run the resolver in parallel with the reader.
This patch makes File::parse() multi-thread safe. If one thread is running
File::parse(), other threads will block if they try to call the same method.
File::parse() is idempotent, so you can safely call multiple times.
With this change, we don't have to wait for all worker threads to finish
in Driver::link(). Previously, Driver::link() calls TaskGroup::sync() to
wait for all threads running File::parse(). This was not ideal because
we couldn't start the resolver until we parse all files.
This patch increase parallelism by making Driver::link() to not wait for
worker threads. The resolver calls parse() to make sure that the file
being read has been parsed, and then uses the file. In this approach,
the resolver can run with the parser threads in parallel.
http://reviews.llvm.org/D6994
Modified:
lld/trunk/include/lld/Core/File.h
lld/trunk/include/lld/Core/LinkingContext.h
lld/trunk/include/lld/Core/Resolver.h
lld/trunk/lib/Core/File.cpp
lld/trunk/lib/Core/Resolver.cpp
lld/trunk/lib/Driver/Driver.cpp
lld/trunk/lib/ReaderWriter/ELF/ELFFile.h
lld/trunk/lib/ReaderWriter/PECOFF/PECOFFLinkingContext.cpp
Modified: lld/trunk/include/lld/Core/File.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/include/lld/Core/File.h?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/include/lld/Core/File.h (original)
+++ lld/trunk/include/lld/Core/File.h Fri Jan 16 09:54:13 2015
@@ -20,6 +20,7 @@
#include "llvm/Support/ErrorHandling.h"
#include <functional>
#include <memory>
+#include <mutex>
#include <vector>
namespace lld {
@@ -161,11 +162,7 @@ public:
/// (because YAML reader does not read blobs but structured data).
void setLastError(std::error_code err) { _lastError = err; }
- std::error_code parse() {
- if (!_lastError.hasValue())
- _lastError = doParse();
- return _lastError.getValue();
- }
+ std::error_code parse();
// Usually each file owns a std::unique_ptr<MemoryBuffer>.
// However, there's one special case. If a file is an archive file,
@@ -239,7 +236,6 @@ protected:
static atom_collection_empty<UndefinedAtom> _noUndefinedAtoms;
static atom_collection_empty<SharedLibraryAtom> _noSharedLibraryAtoms;
static atom_collection_empty<AbsoluteAtom> _noAbsoluteAtoms;
- llvm::Optional<std::error_code> _lastError;
mutable llvm::BumpPtrAllocator _allocator;
private:
@@ -247,6 +243,8 @@ private:
Kind _kind;
mutable uint64_t _ordinal;
std::shared_ptr<MemoryBuffer> _sharedMemoryBuffer;
+ llvm::Optional<std::error_code> _lastError;
+ std::mutex _parseMutex;
};
/// \brief A mutable File.
Modified: lld/trunk/include/lld/Core/LinkingContext.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/include/lld/Core/LinkingContext.h?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/include/lld/Core/LinkingContext.h (original)
+++ lld/trunk/include/lld/Core/LinkingContext.h Fri Jan 16 09:54:13 2015
@@ -13,6 +13,7 @@
#include "lld/Core/Error.h"
#include "lld/Core/LLVM.h"
#include "lld/Core/Node.h"
+#include "lld/Core/Parallel.h"
#include "lld/Core/Reference.h"
#include "lld/Core/range.h"
#include "lld/ReaderWriter/Reader.h"
@@ -323,6 +324,8 @@ public:
// Derived classes may use that chance to rearrange the input files.
virtual void maybeSortInputFiles() {}
+ TaskGroup &getTaskGroup() { return _taskGroup; }
+
/// @}
protected:
LinkingContext(); // Must be subclassed
@@ -370,6 +373,7 @@ protected:
private:
/// Validate the subclass bits. Only called by validate.
virtual bool validateImpl(raw_ostream &diagnostics) = 0;
+ TaskGroup _taskGroup;
};
} // end namespace lld
Modified: lld/trunk/include/lld/Core/Resolver.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/include/lld/Core/Resolver.h?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/include/lld/Core/Resolver.h (original)
+++ lld/trunk/include/lld/Core/Resolver.h Fri Jan 16 09:54:13 2015
@@ -62,7 +62,7 @@ private:
void maybeAddSectionGroupOrGnuLinkOnce(const DefinedAtom &atom);
/// \brief The main function that iterates over the files to resolve
- void resolveUndefines();
+ bool resolveUndefines();
void updateReferences();
void deadStripOptimize();
bool checkUndefines();
Modified: lld/trunk/lib/Core/File.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/lib/Core/File.cpp?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/lib/Core/File.cpp (original)
+++ lld/trunk/lib/Core/File.cpp Fri Jan 16 09:54:13 2015
@@ -9,6 +9,7 @@
#include "lld/Core/File.h"
#include "lld/Core/LLVM.h"
+#include <mutex>
namespace lld {
@@ -19,4 +20,11 @@ File::atom_collection_empty<UndefinedAto
File::atom_collection_empty<SharedLibraryAtom> File::_noSharedLibraryAtoms;
File::atom_collection_empty<AbsoluteAtom> File::_noAbsoluteAtoms;
+std::error_code File::parse() {
+ std::lock_guard<std::mutex> lock(_parseMutex);
+ if (!_lastError.hasValue())
+ _lastError = doParse();
+ return _lastError.getValue();
+}
+
} // namespace lld
Modified: lld/trunk/lib/Core/Resolver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/lib/Core/Resolver.cpp?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/lib/Core/Resolver.cpp (original)
+++ lld/trunk/lib/Core/Resolver.cpp Fri Jan 16 09:54:13 2015
@@ -236,7 +236,7 @@ bool Resolver::undefinesAdded(int begin,
for (int i = begin; i < end; ++i)
if (FileNode *node = dyn_cast<FileNode>(inputs[i].get()))
if (_newUndefinesAdded[node->getFile()])
- return true;
+ return true;
return false;
}
@@ -263,7 +263,7 @@ File *Resolver::getFile(int &index, int
// Keep adding atoms until _context.getNextFile() returns an error. This
// function is where undefined atoms are resolved.
-void Resolver::resolveUndefines() {
+bool Resolver::resolveUndefines() {
ScopedTask task(getDefaultDomain(), "resolveUndefines");
int index = 0;
int groupLevel = 0;
@@ -271,7 +271,12 @@ void Resolver::resolveUndefines() {
bool undefAdded = false;
File *file = getFile(index, groupLevel);
if (!file)
- return;
+ return true;
+ if (std::error_code ec = file->parse()) {
+ llvm::errs() << "Cannot open " + file->path()
+ << ": " << ec.message() << "\n";
+ return false;
+ }
switch (file->kind()) {
case File::kindObject:
if (groupLevel > 0)
@@ -441,7 +446,8 @@ void Resolver::removeCoalescedAwayAtoms(
}
bool Resolver::resolve() {
- resolveUndefines();
+ if (!resolveUndefines())
+ return false;
updateReferences();
deadStripOptimize();
if (checkUndefines())
Modified: lld/trunk/lib/Driver/Driver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/lib/Driver/Driver.cpp?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/lib/Driver/Driver.cpp (original)
+++ lld/trunk/lib/Driver/Driver.cpp Fri Jan 16 09:54:13 2015
@@ -77,42 +77,9 @@ bool Driver::link(LinkingContext &contex
if (context.getNodes().empty())
return false;
- bool fail = false;
-
- // Read inputs
- ScopedTask readTask(getDefaultDomain(), "Read Args");
- TaskGroup tg;
- std::mutex diagnosticsMutex;
- for (std::unique_ptr<Node> &ie : context.getNodes()) {
- tg.spawn([&] {
- // Writes to the same output stream is not guaranteed to be thread-safe.
- // We buffer the diagnostics output to a separate string-backed output
- // stream, acquire the lock, and then print it out.
- std::string buf;
- llvm::raw_string_ostream stream(buf);
-
- if (FileNode *node = dyn_cast<FileNode>(ie.get())) {
- if (File *file = node->getFile()) {
- if (std::error_code ec = file->parse()) {
- stream << "Cannot open " + file->path()
- << ": " << ec.message() << "\n";
- fail = true;
- }
- }
- }
-
- stream.flush();
- if (!buf.empty()) {
- std::lock_guard<std::mutex> lock(diagnosticsMutex);
- diagnostics << buf;
- }
- });
- }
- tg.sync();
- readTask.end();
-
- if (fail)
- return false;
+ for (std::unique_ptr<Node> &ie : context.getNodes())
+ if (FileNode *node = dyn_cast<FileNode>(ie.get()))
+ context.getTaskGroup().spawn([node] { node->getFile()->parse(); });
std::vector<std::unique_ptr<File>> internalFiles;
context.createInternalFiles(internalFiles);
Modified: lld/trunk/lib/ReaderWriter/ELF/ELFFile.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/lib/ReaderWriter/ELF/ELFFile.h?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/lib/ReaderWriter/ELF/ELFFile.h (original)
+++ lld/trunk/lib/ReaderWriter/ELF/ELFFile.h Fri Jan 16 09:54:13 2015
@@ -116,7 +116,9 @@ template <class ELFT> class ELFFile : pu
public:
ELFFile(StringRef name)
- : File(name, kindObject), _ordinal(0), _doStringsMerge(false) {}
+ : File(name, kindObject), _ordinal(0), _doStringsMerge(false) {
+ setLastError(std::error_code());
+ }
ELFFile(std::unique_ptr<MemoryBuffer> mb, bool atomizeStrings = false)
: File(mb->getBufferIdentifier(), kindObject), _mb(std::move(mb)),
Modified: lld/trunk/lib/ReaderWriter/PECOFF/PECOFFLinkingContext.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/lib/ReaderWriter/PECOFF/PECOFFLinkingContext.cpp?rev=226281&r1=226280&r2=226281&view=diff
==============================================================================
--- lld/trunk/lib/ReaderWriter/PECOFF/PECOFFLinkingContext.cpp (original)
+++ lld/trunk/lib/ReaderWriter/PECOFF/PECOFFLinkingContext.cpp Fri Jan 16 09:54:13 2015
@@ -87,6 +87,13 @@ std::unique_ptr<File> PECOFFLinkingConte
"<command line option /include>");
}
+static int getGroupStartPos(std::vector<std::unique_ptr<Node>> &nodes) {
+ for (int i = 0, e = nodes.size(); i < e; ++i)
+ if (GroupEnd *group = dyn_cast<GroupEnd>(nodes[i].get()))
+ return i - group->getSize();
+ llvm::report_fatal_error("internal error");
+}
+
void PECOFFLinkingContext::addLibraryFile(std::unique_ptr<FileNode> file) {
GroupEnd *currentGroupEnd;
int pos = -1;
@@ -111,7 +118,7 @@ bool PECOFFLinkingContext::createImplici
// Create a file for the entry point function.
std::unique_ptr<FileNode> entry(new FileNode(
llvm::make_unique<pecoff::EntryPointFile>(*this, syms)));
- members.insert(members.begin(), std::move(entry));
+ members.insert(members.begin() + getGroupStartPos(members), std::move(entry));
// Create a file for __ImageBase.
std::unique_ptr<FileNode> fileNode(new FileNode(
@@ -339,6 +346,8 @@ void pecoff::ResolvableSymbols::add(File
void pecoff::ResolvableSymbols::readAllSymbols() {
std::lock_guard<std::mutex> lock(_mutex);
for (File *file : _queue) {
+ if (file->parse())
+ return;
if (auto *archive = dyn_cast<ArchiveLibraryFile>(file)) {
for (const std::string &sym : archive->getDefinedSymbols())
_defined.insert(sym);
More information about the llvm-commits
mailing list