[llvm-branch-commits] [clang-tools-extra-branch] r369879 - Merging r369349:
Hans Wennborg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 26 00:56:27 PDT 2019
Author: hans
Date: Mon Aug 26 00:56:27 2019
New Revision: 369879
URL: http://llvm.org/viewvc/llvm-project?rev=369879&view=rev
Log:
Merging r369349:
------------------------------------------------------------------------
r369349 | ibiryukov | 2019-08-20 10:54:30 +0200 (Tue, 20 Aug 2019) | 36 lines
[clangd] Skip function bodies inside processed files while indexing
Summary:
This significantly improves performance of background indexing.
We do not collect references and declarations inside the processed
files, so this does not affect the final indexing results.
The idea is borrowed from libclang, which has a similar optimization in
its indexing functionality.
Measurements show a nice decrease in indexing time, up to ~40% for
building the whole index. These are not proper benchmarks, so one should
not rely on these results too much.
1. Rebuilding the whole index for LLVM:
- Before. Total time: 14m58s.
./bin/clangd -pch-storage=memory < ./clangd.input 23917.67s user 515.86s system 2718% cpu 14:58.68 total
- After. Total time: 8m41s.
./bin/clangd -pch-storage=memory < ./clangd.input 13627.29s user 288.10s system 2672% cpu 8:40.67 total
2. Rebuilding index after removing shards matching '*clangd*' (case-insensitively):
- Before. Total time: 30s.
./bin/clangd -pch-storage=memory < ./clangd.input 130.94s user 6.82s system 452% cpu 30.423 total
- After. Total time: 26s.
./bin/clangd -pch-storage=memory < ./clangd.input 80.51s user 5.40s system 333% cpu 25.777 total
Reviewers: kadircet, sammccall
Reviewed By: kadircet
Subscribers: MaskRay, jkorous, arphaman, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D66226
------------------------------------------------------------------------
Modified:
clang-tools-extra/branches/release_90/ (props changed)
clang-tools-extra/branches/release_90/clangd/index/IndexAction.cpp
clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.cpp
clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.h
Propchange: clang-tools-extra/branches/release_90/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Aug 26 00:56:27 2019
@@ -1 +1 @@
-/clang-tools-extra/trunk:366443,366451,366455,366541,366545,366559,366687,366811,366880,366900,366991-366992,367112,367303,367687,368019,368058,368498,368549,368581
+/clang-tools-extra/trunk:366443,366451,366455,366541,366545,366559,366687,366811,366880,366900,366991-366992,367112,367303,367687,368019,368058,368498,368549,368581,369349
Modified: clang-tools-extra/branches/release_90/clangd/index/IndexAction.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/branches/release_90/clangd/index/IndexAction.cpp?rev=369879&r1=369878&r2=369879&view=diff
==============================================================================
--- clang-tools-extra/branches/release_90/clangd/index/IndexAction.cpp (original)
+++ clang-tools-extra/branches/release_90/clangd/index/IndexAction.cpp Mon Aug 26 00:56:27 2019
@@ -11,9 +11,17 @@
#include "Logger.h"
#include "index/Relation.h"
#include "index/SymbolOrigin.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/MultiplexConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/STLExtras.h"
+#include <functional>
+#include <memory>
#include <utility>
namespace clang {
@@ -113,6 +121,40 @@ private:
IncludeGraph &IG;
};
+/// Returns an ASTConsumer that wraps \p Inner and additionally instructs the
+/// parser to skip bodies of functions in the files that should not be
+/// processed.
+static std::unique_ptr<ASTConsumer>
+skipProcessedFunctions(std::unique_ptr<ASTConsumer> Inner,
+ std::function<bool(FileID)> ShouldIndexFile) {
+ class SkipProcessedFunctions : public ASTConsumer {
+ public:
+ SkipProcessedFunctions(std::function<bool(FileID)> FileFilter)
+ : ShouldIndexFile(std::move(FileFilter)), Context(nullptr) {
+ assert(this->ShouldIndexFile);
+ }
+
+ void Initialize(ASTContext &Context) override { this->Context = &Context; }
+ bool shouldSkipFunctionBody(Decl *D) override {
+ assert(Context && "Initialize() was never called.");
+ auto &SM = Context->getSourceManager();
+ auto FID = SM.getFileID(SM.getExpansionLoc(D->getLocation()));
+ if (!FID.isValid())
+ return false;
+ return !ShouldIndexFile(FID);
+ }
+
+ private:
+ std::function<bool(FileID)> ShouldIndexFile;
+ const ASTContext *Context;
+ };
+ std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+ Consumers.push_back(
+ llvm::make_unique<SkipProcessedFunctions>(ShouldIndexFile));
+ Consumers.push_back(std::move(Inner));
+ return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
+}
+
// Wraps the index action and reports index data after each translation unit.
class IndexAction : public WrapperFrontendAction {
public:
@@ -137,7 +179,9 @@ public:
if (IncludeGraphCallback != nullptr)
CI.getPreprocessor().addPPCallbacks(
llvm::make_unique<IncludeGraphCollector>(CI.getSourceManager(), IG));
- return WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+ return skipProcessedFunctions(
+ WrapperFrontendAction::CreateASTConsumer(CI, InFile),
+ [this](FileID FID) { return Collector->shouldIndexFile(FID); });
}
bool BeginInvocation(CompilerInstance &CI) override {
@@ -147,6 +191,10 @@ public:
// Avoids some analyses too. Set in two places as we're late to the party.
CI.getDiagnosticOpts().IgnoreWarnings = true;
CI.getDiagnostics().setIgnoreAllWarnings(true);
+ // Instruct the parser to ask our ASTConsumer if it should skip function
+ // bodies. The ASTConsumer will take care of skipping only functions inside
+ // the files that we have already processed.
+ CI.getFrontendOpts().SkipFunctionBodies = true;
return WrapperFrontendAction::BeginInvocation(CI);
}
Modified: clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.cpp?rev=369879&r1=369878&r2=369879&view=diff
==============================================================================
--- clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.cpp (original)
+++ clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.cpp Mon Aug 26 00:56:27 2019
@@ -146,17 +146,6 @@ getTokenRange(SourceLocation TokLoc, con
CreatePosition(TokLoc.getLocWithOffset(TokenLength))};
}
-bool shouldIndexFile(const SourceManager &SM, FileID FID,
- const SymbolCollector::Options &Opts,
- llvm::DenseMap<FileID, bool> *FilesToIndexCache) {
- if (!Opts.FileFilter)
- return true;
- auto I = FilesToIndexCache->try_emplace(FID);
- if (I.second)
- I.first->second = Opts.FileFilter(SM, FID);
- return I.first->second;
-}
-
// Return the symbol location of the token at \p TokLoc.
llvm::Optional<SymbolLocation>
getTokenLocation(SourceLocation TokLoc, const SourceManager &SM,
@@ -409,7 +398,7 @@ bool SymbolCollector::handleMacroOccuren
S.SymInfo = index::getSymbolInfoForMacro(*MI);
std::string FileURI;
// FIXME: use the result to filter out symbols.
- shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ shouldIndexFile(SM.getFileID(Loc));
if (auto DeclLoc =
getTokenLocation(DefLoc, SM, Opts, PP->getLangOpts(), FileURI))
S.CanonicalDeclaration = *DeclLoc;
@@ -539,7 +528,7 @@ void SymbolCollector::finish() {
for (const auto &LocAndRole : It.second) {
auto FileID = SM.getFileID(LocAndRole.first);
// FIXME: use the result to filter out references.
- shouldIndexFile(SM, FileID, Opts, &FilesToIndexCache);
+ shouldIndexFile(FileID);
if (auto FileURI = GetURI(FileID)) {
auto Range =
getTokenRange(LocAndRole.first, SM, ASTCtx->getLangOpts());
@@ -589,7 +578,7 @@ const Symbol *SymbolCollector::addDeclar
auto Loc = findNameLoc(&ND);
assert(Loc.isValid() && "Invalid source location for NamedDecl");
// FIXME: use the result to filter out symbols.
- shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ shouldIndexFile(SM.getFileID(Loc));
if (auto DeclLoc =
getTokenLocation(Loc, SM, Opts, ASTCtx->getLangOpts(), FileURI))
S.CanonicalDeclaration = *DeclLoc;
@@ -649,7 +638,7 @@ void SymbolCollector::addDefinition(cons
auto Loc = findNameLoc(&ND);
const auto &SM = ND.getASTContext().getSourceManager();
// FIXME: use the result to filter out symbols.
- shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ shouldIndexFile(SM.getFileID(Loc));
if (auto DefLoc =
getTokenLocation(Loc, SM, Opts, ASTCtx->getLangOpts(), FileURI))
S.Definition = *DefLoc;
@@ -741,5 +730,14 @@ bool SymbolCollector::isDontIncludeMeHea
return false;
}
+bool SymbolCollector::shouldIndexFile(FileID FID) {
+ if (!Opts.FileFilter)
+ return true;
+ auto I = FilesToIndexCache.try_emplace(FID);
+ if (I.second)
+ I.first->second = Opts.FileFilter(ASTCtx->getSourceManager(), FID);
+ return I.first->second;
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.h?rev=369879&r1=369878&r2=369879&view=diff
==============================================================================
--- clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.h (original)
+++ clang-tools-extra/branches/release_90/clangd/index/SymbolCollector.h Mon Aug 26 00:56:27 2019
@@ -112,6 +112,11 @@ public:
RefSlab takeRefs() { return std::move(Refs).build(); }
RelationSlab takeRelations() { return std::move(Relations).build(); }
+ /// Returns true if we are interested in references and declarations from \p
+ /// FID. If this function return false, bodies of functions inside those files
+ /// will be skipped to decrease indexing time.
+ bool shouldIndexFile(FileID FID);
+
void finish() override;
private:
More information about the llvm-branch-commits
mailing list