[clang-tools-extra] r368206 - [clang-doc] Parallelize reducing phase
Diego Astiazaran via cfe-commits
cfe-commits at lists.llvm.org
Wed Aug 7 13:54:26 PDT 2019
Author: diegoastiazaran
Date: Wed Aug 7 13:54:26 2019
New Revision: 368206
URL: http://llvm.org/viewvc/llvm-project?rev=368206&view=rev
Log:
[clang-doc] Parallelize reducing phase
Reduce phase has been parallelized and a execution time was reduced by
60% with this.
The reading of bitcode (bitcode -> Info) was moved to this segment of
code parallelized so it now happens just before reducing.
Differential Revision: https://reviews.llvm.org/D65628
Modified:
clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
Modified: clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp?rev=368206&r1=368205&r2=368206&view=diff
==============================================================================
--- clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp (original)
+++ clang-tools-extra/trunk/clang-doc/tool/ClangDocMain.cpp Wed Aug 7 13:54:26 2019
@@ -28,6 +28,7 @@
#include "clang/ASTMatchers/ASTMatchersInternal.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
@@ -38,7 +39,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
+#include <atomic>
#include <string>
using namespace clang::ast_matchers;
@@ -158,30 +161,6 @@ llvm::Expected<llvm::SmallString<128>> g
return Path;
}
-// Iterate through tool results and build string map of info vectors from the
-// encoded bitstreams.
-bool bitcodeResultsToInfos(
- tooling::ToolResults &Results,
- llvm::StringMap<std::vector<std::unique_ptr<doc::Info>>> &Output) {
- bool Err = false;
- Results.forEachResult([&](StringRef Key, StringRef Value) {
- llvm::BitstreamCursor Stream(Value);
- doc::ClangDocBitcodeReader Reader(Stream);
- auto Infos = Reader.readBitcode();
- if (!Infos) {
- llvm::errs() << toString(Infos.takeError()) << "\n";
- Err = true;
- return;
- }
- for (auto &I : Infos.get()) {
- auto R =
- Output.try_emplace(Key, std::vector<std::unique_ptr<doc::Info>>());
- R.first->second.emplace_back(std::move(I));
- }
- });
- return Err;
-}
-
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
std::error_code OK;
@@ -256,40 +235,72 @@ int main(int argc, const char **argv) {
// In ToolResults, the Key is the hashed USR and the value is the
// bitcode-encoded representation of the Info object.
llvm::outs() << "Collecting infos...\n";
- llvm::StringMap<std::vector<std::unique_ptr<doc::Info>>> USRToInfos;
- if (bitcodeResultsToInfos(*Exec->get()->getToolResults(), USRToInfos))
- return 1;
+ llvm::StringMap<std::vector<StringRef>> USRToBitcode;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+ auto R = USRToBitcode.try_emplace(Key, std::vector<StringRef>());
+ R.first->second.emplace_back(Value);
+ });
// First reducing phase (reduce all decls into one info per decl).
- llvm::outs() << "Reducing " << USRToInfos.size() << " infos...\n";
- for (auto &Group : USRToInfos) {
- auto Reduced = doc::mergeInfos(Group.getValue());
- if (!Reduced) {
- llvm::errs() << llvm::toString(Reduced.takeError());
- continue;
- }
-
- doc::Info *I = Reduced.get().get();
- auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
- "." + Format);
- if (!InfoPath) {
- llvm::errs() << toString(InfoPath.takeError()) << "\n";
- return 1;
- }
- std::error_code FileErr;
- llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
- llvm::sys::fs::OF_None);
- if (FileErr != OK) {
- llvm::errs() << "Error opening info file: " << FileErr.message() << "\n";
- continue;
- }
+ llvm::outs() << "Reducing " << USRToBitcode.size() << " infos...\n";
+ std::atomic<bool> Error;
+ Error = false;
+ // ExecutorConcurrency is a flag exposed by AllTUsExecution.h
+ llvm::ThreadPool Pool(ExecutorConcurrency == 0 ? llvm::hardware_concurrency()
+ : ExecutorConcurrency);
+ for (auto &Group : USRToBitcode) {
+ Pool.async([&]() {
+ std::vector<std::unique_ptr<doc::Info>> Infos;
+
+ for (auto &Bitcode : Group.getValue()) {
+ llvm::BitstreamCursor Stream(Bitcode);
+ doc::ClangDocBitcodeReader Reader(Stream);
+ auto ReadInfos = Reader.readBitcode();
+ if (!ReadInfos) {
+ llvm::errs() << toString(ReadInfos.takeError()) << "\n";
+ Error = true;
+ return;
+ }
+ std::move(ReadInfos->begin(), ReadInfos->end(),
+ std::back_inserter(Infos));
+ }
+
+ auto Reduced = doc::mergeInfos(Infos);
+ if (!Reduced) {
+ llvm::errs() << llvm::toString(Reduced.takeError());
+ return;
+ }
+
+ doc::Info *I = Reduced.get().get();
+ auto InfoPath = getInfoOutputFile(OutDirectory, I->Path, I->extractName(),
+ "." + Format);
+ if (!InfoPath) {
+ llvm::errs() << toString(InfoPath.takeError()) << "\n";
+ Error = true;
+ return;
+ }
+ std::error_code FileErr;
+ llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr,
+ llvm::sys::fs::F_None);
+ if (FileErr != OK) {
+ llvm::errs() << "Error opening info file: " << FileErr.message()
+ << "\n";
+ return;
+ }
+
+ // Add a reference to this Info in the Index
+ clang::doc::Generator::addInfoToIndex(CDCtx.Idx, I);
+
+ if (auto Err = G->get()->generateDocForInfo(I, InfoOS, CDCtx))
+ llvm::errs() << toString(std::move(Err)) << "\n";
+ });
+ }
- // Add a reference to this Info in the Index
- clang::doc::Generator::addInfoToIndex(CDCtx.Idx, I);
+ Pool.wait();
- if (auto Err = G->get()->generateDocForInfo(I, InfoOS, CDCtx))
- llvm::errs() << toString(std::move(Err)) << "\n";
- }
+ if (Error)
+ return 1;
llvm::outs() << "Generating assets for docs...\n";
if (!G->get()->createResources(CDCtx))
More information about the cfe-commits
mailing list