[llvm] r356178 - Speeding up llvm-cov export with multithreaded renderFiles implementation.
Max Moroz via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 14 10:49:27 PDT 2019
Author: dor1s
Date: Thu Mar 14 10:49:27 2019
New Revision: 356178
URL: http://llvm.org/viewvc/llvm-project?rev=356178&view=rev
Log:
Speeding up llvm-cov export with multithreaded renderFiles implementation.
Summary:
CoverageExporterJson::renderFiles accounts for most of the execution time given a large profdata file with multiple binaries.
Proposed solution is to generate JSON for each file in parallel and sort at the end to preserve deterministic output. Also added flags to skip generating parts of the output to trim the output size.
Patch by Sajjad Mirza (@sajjadm).
Reviewers: Dor1s, vsk
Reviewed By: Dor1s, vsk
Subscribers: liaoyuke, mgrang, jdoerfert, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D59277
Added:
llvm/trunk/test/tools/llvm-cov/export_functions.test
Modified:
llvm/trunk/docs/CommandGuide/llvm-cov.rst
llvm/trunk/test/tools/llvm-cov/showExpansions.cpp
llvm/trunk/tools/llvm-cov/CodeCoverage.cpp
llvm/trunk/tools/llvm-cov/CoverageExporterJson.cpp
llvm/trunk/tools/llvm-cov/CoverageViewOptions.h
Modified: llvm/trunk/docs/CommandGuide/llvm-cov.rst
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/docs/CommandGuide/llvm-cov.rst?rev=356178&r1=356177&r2=356178&view=diff
==============================================================================
--- llvm/trunk/docs/CommandGuide/llvm-cov.rst (original)
+++ llvm/trunk/docs/CommandGuide/llvm-cov.rst Thu Mar 14 10:49:27 2019
@@ -419,3 +419,16 @@ OPTIONS
.. option:: -ignore-filename-regex=<PATTERN>
Skip source code files with file paths that match the given regular expression.
+
+ .. option:: -skip-expansions
+
+ Skip exporting macro expansion coverage data.
+
+ .. option:: -skip-functions
+
+ Skip exporting per-function coverage data.
+
+ .. option:: -num-threads=N, -j=N
+
+ Use N threads to export coverage data. When N=0, llvm-cov auto-detects an
+ appropriate number of threads to use. This is the default.
Added: llvm/trunk/test/tools/llvm-cov/export_functions.test
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/llvm-cov/export_functions.test?rev=356178&view=auto
==============================================================================
--- llvm/trunk/test/tools/llvm-cov/export_functions.test (added)
+++ llvm/trunk/test/tools/llvm-cov/export_functions.test Thu Mar 14 10:49:27 2019
@@ -0,0 +1,10 @@
+# Test that llvm-cov export produces function data by default and that it can be
+# turned off with a flag.
+
+RUN: llvm-cov export %S/Inputs/report.covmapping -instr-profile %S/Inputs/report.profdata 2>&1 | FileCheck %s
+RUN: llvm-cov export %S/Inputs/report.covmapping -instr-profile %S/Inputs/report.profdata -skip-functions 2>&1 | FileCheck -check-prefix=SKIP-FUNCTIONS %s
+
+CHECK: "functions":[
+SKIP-FUNCTIONS-NOT: "functions":[
+
+
Modified: llvm/trunk/test/tools/llvm-cov/showExpansions.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/tools/llvm-cov/showExpansions.cpp?rev=356178&r1=356177&r2=356178&view=diff
==============================================================================
--- llvm/trunk/test/tools/llvm-cov/showExpansions.cpp (original)
+++ llvm/trunk/test/tools/llvm-cov/showExpansions.cpp Thu Mar 14 10:49:27 2019
@@ -25,3 +25,6 @@ int main(int argc, const char *argv[]) {
return 0;
}
// RUN: llvm-cov export %S/Inputs/showExpansions.covmapping -instr-profile %S/Inputs/showExpansions.profdata 2>&1 | FileCheck %S/Inputs/showExpansions.json
+
+// RUN: llvm-cov export %S/Inputs/showExpansions.covmapping -instr-profile %S/Inputs/showExpansions.profdata -skip-expansions 2>&1 | FileCheck %s -check-prefix=SKIP-EXPANSIONS
+// SKIP-EXPANSIONS-NOT: "expansions"
Modified: llvm/trunk/tools/llvm-cov/CodeCoverage.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-cov/CodeCoverage.cpp?rev=356178&r1=356177&r2=356178&view=diff
==============================================================================
--- llvm/trunk/tools/llvm-cov/CodeCoverage.cpp (original)
+++ llvm/trunk/tools/llvm-cov/CodeCoverage.cpp Thu Mar 14 10:49:27 2019
@@ -1006,10 +1006,23 @@ int CodeCoverageTool::doReport(int argc,
int CodeCoverageTool::doExport(int argc, const char **argv,
CommandLineParserType commandLineParser) {
+ cl::OptionCategory ExportCategory("Exporting options");
+
+ cl::opt<bool> SkipExpansions("skip-expansions", cl::Optional,
+ cl::desc("Don't export expanded source regions"),
+ cl::cat(ExportCategory));
+
+ cl::opt<bool> SkipFunctions("skip-functions", cl::Optional,
+ cl::desc("Don't export per-function data"),
+ cl::cat(ExportCategory));
+
auto Err = commandLineParser(argc, argv);
if (Err)
return Err;
+ ViewOpts.SkipExpansions = SkipExpansions;
+ ViewOpts.SkipFunctions = SkipFunctions;
+
if (ViewOpts.Format != CoverageViewOptions::OutputFormat::Text &&
ViewOpts.Format != CoverageViewOptions::OutputFormat::Lcov) {
error("Coverage data can only be exported as textual JSON or an "
Modified: llvm/trunk/tools/llvm-cov/CoverageExporterJson.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-cov/CoverageExporterJson.cpp?rev=356178&r1=356177&r2=356178&view=diff
==============================================================================
--- llvm/trunk/tools/llvm-cov/CoverageExporterJson.cpp (original)
+++ llvm/trunk/tools/llvm-cov/CoverageExporterJson.cpp Thu Mar 14 10:49:27 2019
@@ -42,7 +42,14 @@
#include "CoverageExporterJson.h"
#include "CoverageReport.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/JSON.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
+#include <algorithm>
+#include <mutex>
+#include <utility>
/// The semantic version combined as a string.
#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.0"
@@ -127,13 +134,15 @@ json::Array renderFileSegments(const cov
json::Object renderFile(const coverage::CoverageMapping &Coverage,
const std::string &Filename,
const FileCoverageSummary &FileReport,
- bool ExportSummaryOnly) {
+ const CoverageViewOptions &Options) {
json::Object File({{"filename", Filename}});
- if (!ExportSummaryOnly) {
+ if (!Options.ExportSummaryOnly) {
// Calculate and render detailed coverage information for given file.
auto FileCoverage = Coverage.getCoverageForFile(Filename);
File["segments"] = renderFileSegments(FileCoverage, FileReport);
- File["expansions"] = renderFileExpansions(FileCoverage, FileReport);
+ if (!Options.SkipExpansions) {
+ File["expansions"] = renderFileExpansions(FileCoverage, FileReport);
+ }
}
File["summary"] = renderSummary(FileReport);
return File;
@@ -142,11 +151,28 @@ json::Object renderFile(const coverage::
json::Array renderFiles(const coverage::CoverageMapping &Coverage,
ArrayRef<std::string> SourceFiles,
ArrayRef<FileCoverageSummary> FileReports,
- bool ExportSummaryOnly) {
+ const CoverageViewOptions &Options) {
+ auto NumThreads = Options.NumThreads;
+ if (NumThreads == 0) {
+ NumThreads = std::max(1U, std::min(llvm::heavyweight_hardware_concurrency(),
+ unsigned(SourceFiles.size())));
+ }
+ ThreadPool Pool(NumThreads);
json::Array FileArray;
- for (unsigned I = 0, E = SourceFiles.size(); I < E; ++I)
- FileArray.push_back(renderFile(Coverage, SourceFiles[I], FileReports[I],
- ExportSummaryOnly));
+ std::mutex FileArrayMutex;
+
+ for (unsigned I = 0, E = SourceFiles.size(); I < E; ++I) {
+ auto &SourceFile = SourceFiles[I];
+ auto &FileReport = FileReports[I];
+ Pool.async([&] {
+ auto File = renderFile(Coverage, SourceFile, FileReport, Options);
+ {
+ std::lock_guard<std::mutex> Lock(FileArrayMutex);
+ FileArray.push_back(std::move(File));
+ }
+ });
+ }
+ Pool.wait();
return FileArray;
}
@@ -177,12 +203,22 @@ void CoverageExporterJson::renderRoot(Ar
FileCoverageSummary Totals = FileCoverageSummary("Totals");
auto FileReports = CoverageReport::prepareFileReports(Coverage, Totals,
SourceFiles, Options);
- auto Export =
- json::Object({{"files", renderFiles(Coverage, SourceFiles, FileReports,
- Options.ExportSummaryOnly)},
- {"totals", renderSummary(Totals)}});
- // Skip functions-level information for summary-only export mode.
- if (!Options.ExportSummaryOnly)
+ auto Files = renderFiles(Coverage, SourceFiles, FileReports, Options);
+ // Sort files in order of their names.
+ std::sort(Files.begin(), Files.end(),
+ [](const json::Value &A, const json::Value &B) {
+ const json::Object *ObjA = A.getAsObject();
+ const json::Object *ObjB = B.getAsObject();
+ assert(ObjA != nullptr && "Value A was not an Object");
+ assert(ObjB != nullptr && "Value B was not an Object");
+ const StringRef FilenameA = ObjA->getString("filename").getValue();
+ const StringRef FilenameB = ObjB->getString("filename").getValue();
+ return FilenameA.compare(FilenameB) < 0;
+ });
+ auto Export = json::Object(
+ {{"files", std::move(Files)}, {"totals", renderSummary(Totals)}});
+ // Skip functions-level information if necessary.
+ if (!Options.ExportSummaryOnly && !Options.SkipFunctions)
Export["functions"] = renderFunctions(Coverage.getCoveredFunctions());
auto ExportArray = json::Array({std::move(Export)});
Modified: llvm/trunk/tools/llvm-cov/CoverageViewOptions.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-cov/CoverageViewOptions.h?rev=356178&r1=356177&r2=356178&view=diff
==============================================================================
--- llvm/trunk/tools/llvm-cov/CoverageViewOptions.h (original)
+++ llvm/trunk/tools/llvm-cov/CoverageViewOptions.h Thu Mar 14 10:49:27 2019
@@ -34,6 +34,8 @@ struct CoverageViewOptions {
bool ShowRegionSummary;
bool ShowInstantiationSummary;
bool ExportSummaryOnly;
+ bool SkipExpansions;
+ bool SkipFunctions;
OutputFormat Format;
std::string ShowOutputDirectory;
std::vector<std::string> DemanglerOpts;
More information about the llvm-commits
mailing list