[PATCH] D19720: [find-all-symbols] Parallelize the merge step.
Benjamin Kramer via cfe-commits
cfe-commits at lists.llvm.org
Fri Apr 29 05:05:11 PDT 2016
bkramer created this revision.
bkramer added reviewers: hokein, djasper.
bkramer added a subscriber: cfe-commits.
There is still more parallelism to get here because we synchonize on the
actual uniquing but just doing YAML parsing in parallel already gives a
significant speedup.
Merging all symbols in LLVM+clang+compiler-rt+lld+libc++, 48 cores.
before: 201.55s user 1.47s system 99% cpu 3:23.04 total
after: 279.13s user 7.53s system 929% cpu 30.838 total
http://reviews.llvm.org/D19720
Files:
include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
Index: include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
===================================================================
--- include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
+++ include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp
@@ -14,8 +14,9 @@
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/Support/Path.h"
-
+#include "llvm/Support/ThreadPool.h"
#include <map>
+#include <mutex>
#include <string>
#include <vector>
@@ -74,23 +75,34 @@
};
bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
- std::error_code EC;
std::set<SymbolInfo> UniqueSymbols;
+ std::mutex SymbolMutex;
+ auto AddSymbols = [&](ArrayRef<SymbolInfo> Symbols) {
+ // Synchronize set accesses.
+ std::unique_lock<std::mutex> LockGuard(SymbolMutex);
+ for (const SymbolInfo &Symbol : Symbols)
+ UniqueSymbols.insert(Symbol);
+ };
+
// Load all symbol files in MergeDir.
+ llvm::ThreadPool Pool;
+ std::error_code EC;
for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd;
Dir != DirEnd && !EC; Dir.increment(EC)) {
- int ReadFD = 0;
- if (llvm::sys::fs::openFileForRead(Dir->path(), ReadFD)) {
- llvm::errs() << "Cann't open " << Dir->path() << "\n";
- continue;
- }
- auto Buffer = llvm::MemoryBuffer::getOpenFile(ReadFD, Dir->path(), -1);
- if (!Buffer)
- continue;
- std::vector<SymbolInfo> Symbols =
- ReadSymbolInfosFromYAML(Buffer.get()->getBuffer());
- for (const auto &Symbol : Symbols)
- UniqueSymbols.insert(Symbol);
+ // Do the YAML file parsing in parallel.
+ Pool.async(
+ [&AddSymbols](std::string Path) {
+ auto Buffer = llvm::MemoryBuffer::getFile(Path);
+ if (!Buffer) {
+ llvm::errs() << "Cann't open " << Path << "\n";
+ return;
+ }
+ std::vector<SymbolInfo> Symbols =
+ ReadSymbolInfosFromYAML(Buffer.get()->getBuffer());
+ // FIXME: Merge without creating such a heavy contention point.
+ AddSymbols(Symbols);
+ },
+ Dir->path());
}
llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::F_None);
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D19720.55575.patch
Type: text/x-patch
Size: 2216 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20160429/3eaacaea/attachment.bin>
More information about the cfe-commits
mailing list