<div dir="ltr">This broke buildbots. Sorry about that.<div>r333742 should fix them.</div></div><br><div class="gmail_quote"><div dir="ltr">On Fri, Jun 1, 2018 at 12:12 PM Ilya Biryukov via cfe-commits <<a href="mailto:cfe-commits@lists.llvm.org" target="_blank">cfe-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: ibiryukov<br>
Date: Fri Jun 1 03:08:43 2018<br>
New Revision: 333737<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=333737&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=333737&view=rev</a><br>
Log:<br>
[clangd] Keep only a limited number of idle ASTs in memory<br>
<br>
Summary:<br>
After this commit, clangd will only keep the last 3 accessed ASTs in<br>
memory. Preambles for each of the opened files are still kept in<br>
memory to make completion and AST rebuilds fast.<br>
<br>
AST rebuilds are usually fast enough, but having the last ASTs in<br>
memory still considerably improves latency of operations like<br>
findDefinition and documeneHighlight, which are often sent multiple<br>
times a second when moving around the code. So keeping some of the last<br>
accessed ASTs in memory seems like a reasonable tradeoff.<br>
<br>
Reviewers: sammccall<br>
<br>
Reviewed By: sammccall<br>
<br>
Subscribers: malaperle, arphaman, klimek, javed.absar, ioeric, MaskRay, jkorous, cfe-commits<br>
<br>
Differential Revision: <a href="https://reviews.llvm.org/D47063" rel="noreferrer" target="_blank">https://reviews.llvm.org/D47063</a><br>
<br>
Modified:<br>
clang-tools-extra/trunk/clangd/ClangdServer.cpp<br>
clang-tools-extra/trunk/clangd/ClangdServer.h<br>
clang-tools-extra/trunk/clangd/ClangdUnit.cpp<br>
clang-tools-extra/trunk/clangd/ClangdUnit.h<br>
clang-tools-extra/trunk/clangd/TUScheduler.cpp<br>
clang-tools-extra/trunk/clangd/TUScheduler.h<br>
clang-tools-extra/trunk/test/clangd/trace.test<br>
clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp<br>
clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp<br>
<br>
Modified: clang-tools-extra/trunk/clangd/ClangdServer.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/ClangdServer.cpp (original)<br>
+++ clang-tools-extra/trunk/clangd/ClangdServer.cpp Fri Jun 1 03:08:43 2018<br>
@@ -100,7 +100,7 @@ ClangdServer::ClangdServer(GlobalCompila<br>
std::shared_ptr<Preprocessor><br>
PP) { FileIdx->update(Path, &AST, std::move(PP)); }<br>
: PreambleParsedCallback(),<br>
- Opts.UpdateDebounce) {<br>
+ Opts.UpdateDebounce, Opts.RetentionPolicy) {<br>
if (FileIdx && Opts.StaticIndex) {<br>
MergedIndex = mergeIndex(FileIdx.get(), Opts.StaticIndex);<br>
Index = MergedIndex.get();<br>
<br>
Modified: clang-tools-extra/trunk/clangd/ClangdServer.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.h?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.h?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/ClangdServer.h (original)<br>
+++ clang-tools-extra/trunk/clangd/ClangdServer.h Fri Jun 1 03:08:43 2018<br>
@@ -70,6 +70,9 @@ public:<br>
/// If 0, all requests are processed on the calling thread.<br>
unsigned AsyncThreadsCount = getDefaultAsyncThreadsCount();<br>
<br>
+ /// AST caching policy. The default is to keep up to 3 ASTs in memory.<br>
+ ASTRetentionPolicy RetentionPolicy;<br>
+<br>
/// Cached preambles are potentially large. If false, store them on disk.<br>
bool StorePreamblesInMemory = true;<br>
<br>
<br>
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original)<br>
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Fri Jun 1 03:08:43 2018<br>
@@ -175,8 +175,12 @@ ParsedAST::Build(std::unique_ptr<clang::<br>
ASTDiags.EndSourceFile();<br>
<br>
std::vector<const Decl *> ParsedDecls = Action->takeTopLevelDecls();<br>
+ std::vector<Diag> Diags = ASTDiags.take();<br>
+ // Add diagnostics from the preamble, if any.<br>
+ if (Preamble)<br>
+ Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());<br>
return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),<br>
- std::move(ParsedDecls), ASTDiags.take(),<br>
+ std::move(ParsedDecls), std::move(Diags),<br>
std::move(Inclusions));<br>
}<br>
<br>
@@ -243,120 +247,57 @@ ParsedAST::ParsedAST(std::shared_ptr<con<br>
assert(this->Action);<br>
}<br>
<br>
-CppFile::CppFile(PathRef FileName, bool StorePreamblesInMemory,<br>
- std::shared_ptr<PCHContainerOperations> PCHs,<br>
- PreambleParsedCallback PreambleCallback)<br>
- : FileName(FileName), StorePreamblesInMemory(StorePreamblesInMemory),<br>
- PCHs(std::move(PCHs)), PreambleCallback(std::move(PreambleCallback)) {<br>
- log("Created CppFile for " + FileName);<br>
-}<br>
-<br>
-llvm::Optional<std::vector<Diag>> CppFile::rebuild(ParseInputs &&Inputs) {<br>
- log("Rebuilding file " + FileName + " with command [" +<br>
- Inputs.CompileCommand.Directory + "] " +<br>
- llvm::join(Inputs.CompileCommand.CommandLine, " "));<br>
-<br>
+std::unique_ptr<CompilerInvocation><br>
+clangd::buildCompilerInvocation(const ParseInputs &Inputs) {<br>
std::vector<const char *> ArgStrs;<br>
for (const auto &S : Inputs.CompileCommand.CommandLine)<br>
ArgStrs.push_back(S.c_str());<br>
<br>
if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {<br>
- log("Couldn't set working directory");<br>
- // We run parsing anyway, our lit-tests rely on results for non-existing<br>
- // working dirs.<br>
- }<br>
-<br>
- // Prepare CompilerInvocation.<br>
- std::unique_ptr<CompilerInvocation> CI;<br>
- {<br>
- // FIXME(ibiryukov): store diagnostics from CommandLine when we start<br>
- // reporting them.<br>
- IgnoreDiagnostics IgnoreDiagnostics;<br>
- IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =<br>
- CompilerInstance::createDiagnostics(new DiagnosticOptions,<br>
- &IgnoreDiagnostics, false);<br>
- CI = createInvocationFromCommandLine(ArgStrs, CommandLineDiagsEngine,<br>
- Inputs.FS);<br>
- if (!CI) {<br>
- log("Could not build CompilerInvocation for file " + FileName);<br>
- AST = llvm::None;<br>
- Preamble = nullptr;<br>
- return llvm::None;<br>
- }<br>
- // createInvocationFromCommandLine sets DisableFree.<br>
- CI->getFrontendOpts().DisableFree = false;<br>
- CI->getLangOpts()->CommentOpts.ParseAllComments = true;<br>
- }<br>
-<br>
- std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =<br>
- llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, FileName);<br>
-<br>
- // Compute updated Preamble.<br>
- std::shared_ptr<const PreambleData> NewPreamble =<br>
- rebuildPreamble(*CI, Inputs.CompileCommand, Inputs.FS, *ContentsBuffer);<br>
-<br>
- // Remove current AST to avoid wasting memory.<br>
- AST = llvm::None;<br>
- // Compute updated AST.<br>
- llvm::Optional<ParsedAST> NewAST;<br>
- {<br>
- trace::Span Tracer("Build");<br>
- SPAN_ATTACH(Tracer, "File", FileName);<br>
- NewAST = ParsedAST::Build(std::move(CI), NewPreamble,<br>
- std::move(ContentsBuffer), PCHs, Inputs.FS);<br>
- }<br>
-<br>
- std::vector<Diag> Diagnostics;<br>
- if (NewAST) {<br>
- // Collect diagnostics from both the preamble and the AST.<br>
- if (NewPreamble)<br>
- Diagnostics = NewPreamble->Diags;<br>
- Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(),<br>
- NewAST->getDiagnostics().end());<br>
- }<br>
-<br>
- // Write the results of rebuild into class fields.<br>
- Command = std::move(Inputs.CompileCommand);<br>
- Preamble = std::move(NewPreamble);<br>
- AST = std::move(NewAST);<br>
- return Diagnostics;<br>
-}<br>
-<br>
-const std::shared_ptr<const PreambleData> &CppFile::getPreamble() const {<br>
- return Preamble;<br>
-}<br>
-<br>
-ParsedAST *CppFile::getAST() const {<br>
- // We could add mutable to AST instead of const_cast here, but that would also<br>
- // allow writing to AST from const methods.<br>
- return AST ? const_cast<ParsedAST *>(AST.getPointer()) : nullptr;<br>
-}<br>
-<br>
-std::size_t CppFile::getUsedBytes() const {<br>
- std::size_t Total = 0;<br>
- if (AST)<br>
- Total += AST->getUsedBytes();<br>
- if (StorePreamblesInMemory && Preamble)<br>
- Total += Preamble->Preamble.getSize();<br>
- return Total;<br>
-}<br>
-<br>
-std::shared_ptr<const PreambleData><br>
-CppFile::rebuildPreamble(CompilerInvocation &CI,<br>
- const tooling::CompileCommand &Command,<br>
- IntrusiveRefCntPtr<vfs::FileSystem> FS,<br>
- llvm::MemoryBuffer &ContentsBuffer) const {<br>
- const auto &OldPreamble = this->Preamble;<br>
- auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), &ContentsBuffer, 0);<br>
- if (OldPreamble && compileCommandsAreEqual(this->Command, Command) &&<br>
- OldPreamble->Preamble.CanReuse(CI, &ContentsBuffer, Bounds, FS.get())) {<br>
+ log("Couldn't set working directory when creating compiler invocation.");<br>
+ // We proceed anyway, our lit-tests rely on results for non-existing working<br>
+ // dirs.<br>
+ }<br>
+<br>
+ // FIXME(ibiryukov): store diagnostics from CommandLine when we start<br>
+ // reporting them.<br>
+ IgnoreDiagnostics IgnoreDiagnostics;<br>
+ IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =<br>
+ CompilerInstance::createDiagnostics(new DiagnosticOptions,<br>
+ &IgnoreDiagnostics, false);<br>
+ std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine(<br>
+ ArgStrs, CommandLineDiagsEngine, Inputs.FS);<br>
+ if (!CI)<br>
+ return nullptr;<br>
+ // createInvocationFromCommandLine sets DisableFree.<br>
+ CI->getFrontendOpts().DisableFree = false;<br>
+ CI->getLangOpts()->CommentOpts.ParseAllComments = true;<br>
+ return CI;<br>
+}<br>
+<br>
+std::shared_ptr<const PreambleData> clangd::buildPreamble(<br>
+ PathRef FileName, CompilerInvocation &CI,<br>
+ std::shared_ptr<const PreambleData> OldPreamble,<br>
+ const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs, bool StoreInMemory,<br>
+ PreambleParsedCallback PreambleCallback) {<br>
+ // Note that we don't need to copy the input contents, preamble can live<br>
+ // without those.<br>
+ auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents);<br>
+ auto Bounds =<br>
+ ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);<br>
+<br>
+ if (OldPreamble &&<br>
+ compileCommandsAreEqual(Inputs.CompileCommand, OldCompileCommand) &&<br>
+ OldPreamble->Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds,<br>
+ Inputs.FS.get())) {<br>
log("Reusing preamble for file " + Twine(FileName));<br>
return OldPreamble;<br>
}<br>
log("Preamble for file " + Twine(FileName) +<br>
" cannot be reused. Attempting to rebuild it.");<br>
<br>
- trace::Span Tracer("Preamble");<br>
+ trace::Span Tracer("BuildPreamble");<br>
SPAN_ATTACH(Tracer, "File", FileName);<br>
StoreDiags PreambleDiagnostics;<br>
IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =<br>
@@ -369,9 +310,14 @@ CppFile::rebuildPreamble(CompilerInvocat<br>
CI.getFrontendOpts().SkipFunctionBodies = true;<br>
<br>
CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);<br>
+ if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {<br>
+ log("Couldn't set working directory when building the preamble.");<br>
+ // We proceed anyway, our lit-tests rely on results for non-existing working<br>
+ // dirs.<br>
+ }<br>
auto BuiltPreamble = PrecompiledPreamble::Build(<br>
- CI, &ContentsBuffer, Bounds, *PreambleDiagsEngine, FS, PCHs,<br>
- /*StoreInMemory=*/StorePreamblesInMemory, SerializedDeclsCollector);<br>
+ CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Inputs.FS, PCHs,<br>
+ StoreInMemory, SerializedDeclsCollector);<br>
<br>
// When building the AST for the main file, we do want the function<br>
// bodies.<br>
@@ -380,7 +326,6 @@ CppFile::rebuildPreamble(CompilerInvocat<br>
if (BuiltPreamble) {<br>
log("Built preamble of size " + Twine(BuiltPreamble->getSize()) +<br>
" for file " + Twine(FileName));<br>
-<br>
return std::make_shared<PreambleData>(<br>
std::move(*BuiltPreamble), PreambleDiagnostics.take(),<br>
SerializedDeclsCollector.takeInclusions());<br>
@@ -390,6 +335,24 @@ CppFile::rebuildPreamble(CompilerInvocat<br>
}<br>
}<br>
<br>
+llvm::Optional<ParsedAST> clangd::buildAST(<br>
+ PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,<br>
+ const ParseInputs &Inputs, std::shared_ptr<const PreambleData> Preamble,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs) {<br>
+ trace::Span Tracer("BuildAST");<br>
+ SPAN_ATTACH(Tracer, "File", FileName);<br>
+<br>
+ if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {<br>
+ log("Couldn't set working directory when building the preamble.");<br>
+ // We proceed anyway, our lit-tests rely on results for non-existing working<br>
+ // dirs.<br>
+ }<br>
+<br>
+ return ParsedAST::Build(<br>
+ llvm::make_unique<CompilerInvocation>(*Invocation), Preamble,<br>
+ llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), PCHs, Inputs.FS);<br>
+}<br>
+<br>
SourceLocation clangd::getBeginningOfIdentifier(ParsedAST &Unit,<br>
const Position &Pos,<br>
const FileID FID) {<br>
<br>
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/ClangdUnit.h (original)<br>
+++ clang-tools-extra/trunk/clangd/ClangdUnit.h Fri Jun 1 03:08:43 2018<br>
@@ -47,6 +47,7 @@ struct PreambleData {<br>
PreambleData(PrecompiledPreamble Preamble, std::vector<Diag> Diags,<br>
std::vector<Inclusion> Inclusions);<br>
<br>
+ tooling::CompileCommand CompileCommand;<br>
PrecompiledPreamble Preamble;<br>
std::vector<Diag> Diags;<br>
// Processes like code completions and go-to-definitions will need #include<br>
@@ -128,50 +129,32 @@ private:<br>
using PreambleParsedCallback = std::function<void(<br>
PathRef Path, ASTContext &, std::shared_ptr<clang::Preprocessor>)>;<br>
<br>
-/// Manages resources, required by clangd. Allows to rebuild file with new<br>
-/// contents, and provides AST and Preamble for it.<br>
-class CppFile {<br>
-public:<br>
- CppFile(PathRef FileName, bool StorePreamblesInMemory,<br>
- std::shared_ptr<PCHContainerOperations> PCHs,<br>
- PreambleParsedCallback PreambleCallback);<br>
-<br>
- /// Rebuild the AST and the preamble.<br>
- /// Returns a list of diagnostics or llvm::None, if an error occured.<br>
- llvm::Optional<std::vector<Diag>> rebuild(ParseInputs &&Inputs);<br>
- /// Returns the last built preamble.<br>
- const std::shared_ptr<const PreambleData> &getPreamble() const;<br>
- /// Returns the last built AST.<br>
- ParsedAST *getAST() const;<br>
- /// Returns an estimated size, in bytes, currently occupied by the AST and the<br>
- /// Preamble.<br>
- std::size_t getUsedBytes() const;<br>
-<br>
-private:<br>
- /// Build a new preamble for \p Inputs. If the current preamble can be reused,<br>
- /// it is returned instead.<br>
- /// This method is const to ensure we don't incidentally modify any fields.<br>
- std::shared_ptr<const PreambleData><br>
- rebuildPreamble(CompilerInvocation &CI,<br>
- const tooling::CompileCommand &Command,<br>
- IntrusiveRefCntPtr<vfs::FileSystem> FS,<br>
- llvm::MemoryBuffer &ContentsBuffer) const;<br>
-<br>
- const Path FileName;<br>
- const bool StorePreamblesInMemory;<br>
-<br>
- /// The last CompileCommand used to build AST and Preamble.<br>
- tooling::CompileCommand Command;<br>
- /// The last parsed AST.<br>
- llvm::Optional<ParsedAST> AST;<br>
- /// The last built Preamble.<br>
- std::shared_ptr<const PreambleData> Preamble;<br>
- /// Utility class required by clang<br>
- std::shared_ptr<PCHContainerOperations> PCHs;<br>
- /// This is called after the file is parsed. This can be nullptr if there is<br>
- /// no callback.<br>
- PreambleParsedCallback PreambleCallback;<br>
-};<br>
+/// Builds compiler invocation that could be used to build AST or preamble.<br>
+std::unique_ptr<CompilerInvocation><br>
+buildCompilerInvocation(const ParseInputs &Inputs);<br>
+<br>
+/// Rebuild the preamble for the new inputs unless the old one can be reused.<br>
+/// If \p OldPreamble can be reused, it is returned unchanged.<br>
+/// If \p OldPreamble is null, always builds the preamble.<br>
+/// If \p PreambleCallback is set, it will be run on top of the AST while<br>
+/// building the preamble. Note that if the old preamble was reused, no AST is<br>
+/// built and, therefore, the callback will not be executed.<br>
+std::shared_ptr<const PreambleData><br>
+buildPreamble(PathRef FileName, CompilerInvocation &CI,<br>
+ std::shared_ptr<const PreambleData> OldPreamble,<br>
+ const tooling::CompileCommand &OldCompileCommand,<br>
+ const ParseInputs &Inputs,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs, bool StoreInMemory,<br>
+ PreambleParsedCallback PreambleCallback);<br>
+<br>
+/// Build an AST from provided user inputs. This function does not check if<br>
+/// preamble can be reused, as this function expects that \p Preamble is the<br>
+/// result of calling buildPreamble.<br>
+llvm::Optional<ParsedAST><br>
+buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,<br>
+ const ParseInputs &Inputs,<br>
+ std::shared_ptr<const PreambleData> Preamble,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs);<br>
<br>
/// Get the beginning SourceLocation at a specified \p Pos.<br>
/// May be invalid if Pos is, or if there's no identifier.<br>
<br>
Modified: clang-tools-extra/trunk/clangd/TUScheduler.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/TUScheduler.cpp?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/TUScheduler.cpp?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/TUScheduler.cpp (original)<br>
+++ clang-tools-extra/trunk/clangd/TUScheduler.cpp Fri Jun 1 03:08:43 2018<br>
@@ -45,9 +45,12 @@<br>
#include "TUScheduler.h"<br>
#include "Logger.h"<br>
#include "Trace.h"<br>
+#include "clang/Frontend/CompilerInvocation.h"<br>
#include "clang/Frontend/PCHContainerOperations.h"<br>
+#include "llvm/ADT/ScopeExit.h"<br>
#include "llvm/Support/Errc.h"<br>
#include "llvm/Support/Path.h"<br>
+#include <algorithm><br>
#include <memory><br>
#include <queue><br>
#include <thread><br>
@@ -55,6 +58,76 @@<br>
namespace clang {<br>
namespace clangd {<br>
using std::chrono::steady_clock;<br>
+<br>
+namespace {<br>
+class ASTWorker;<br>
+}<br>
+<br>
+/// An LRU cache of idle ASTs.<br>
+/// Because we want to limit the overall number of these we retain, the cache<br>
+/// owns ASTs (and may evict them) while their workers are idle.<br>
+/// Workers borrow ASTs when active, and return them when done.<br>
+class TUScheduler::ASTCache {<br>
+public:<br>
+ using Key = const ASTWorker *;<br>
+<br>
+ ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}<br>
+<br>
+ /// Returns result of getUsedBytes() for the AST cached by \p K.<br>
+ /// If no AST is cached, 0 is returned.<br>
+ bool getUsedBytes(Key K) {<br>
+ std::lock_guard<std::mutex> Lock(Mut);<br>
+ auto It = findByKey(K);<br>
+ if (It == LRU.end() || !It->second)<br>
+ return 0;<br>
+ return It->second->getUsedBytes();<br>
+ }<br>
+<br>
+ /// Store the value in the pool, possibly removing the last used AST.<br>
+ /// The value should not be in the pool when this function is called.<br>
+ void put(Key K, std::unique_ptr<ParsedAST> V) {<br>
+ std::unique_lock<std::mutex> Lock(Mut);<br>
+ assert(findByKey(K) == LRU.end());<br>
+<br>
+ LRU.insert(LRU.begin(), {K, std::move(V)});<br>
+ if (LRU.size() <= MaxRetainedASTs)<br>
+ return;<br>
+ // We're past the limit, remove the last element.<br>
+ std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);<br>
+ LRU.pop_back();<br>
+ // Run the expensive destructor outside the lock.<br>
+ Lock.unlock();<br>
+ ForCleanup.reset();<br>
+ }<br>
+<br>
+ /// Returns the cached value for \p K, or llvm::None if the value is not in<br>
+ /// the cache anymore. If nullptr was cached for \p K, this function will<br>
+ /// return a null unique_ptr wrapped into an optional.<br>
+ llvm::Optional<std::unique_ptr<ParsedAST>> take(Key K) {<br>
+ std::unique_lock<std::mutex> Lock(Mut);<br>
+ auto Existing = findByKey(K);<br>
+ if (Existing == LRU.end())<br>
+ return llvm::None;<br>
+ std::unique_ptr<ParsedAST> V = std::move(Existing->second);<br>
+ LRU.erase(Existing);<br>
+ return V;<br>
+ }<br>
+<br>
+private:<br>
+ using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;<br>
+<br>
+ std::vector<KVPair>::iterator findByKey(Key K) {<br>
+ return std::find_if(LRU.begin(), LRU.end(),<br>
+ [K](const KVPair &P) { return P.first == K; });<br>
+ }<br>
+<br>
+ std::mutex Mut;<br>
+ unsigned MaxRetainedASTs;<br>
+ /// Items sorted in LRU order, i.e. first item is the most recently accessed<br>
+ /// one.<br>
+ std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */<br>
+};<br>
+<br>
namespace {<br>
class ASTWorkerHandle;<br>
<br>
@@ -70,8 +143,12 @@ class ASTWorkerHandle;<br>
/// worker.<br>
class ASTWorker {<br>
friend class ASTWorkerHandle;<br>
- ASTWorker(llvm::StringRef File, Semaphore &Barrier, CppFile AST, bool RunSync,<br>
- steady_clock::duration UpdateDebounce);<br>
+ ASTWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache,<br>
+ Semaphore &Barrier, bool RunSync,<br>
+ steady_clock::duration UpdateDebounce,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs,<br>
+ bool StorePreamblesInMemory,<br>
+ PreambleParsedCallback PreambleCallback);<br>
<br>
public:<br>
/// Create a new ASTWorker and return a handle to it.<br>
@@ -79,9 +156,13 @@ public:<br>
/// is null, all requests will be processed on the calling thread<br>
/// synchronously instead. \p Barrier is acquired when processing each<br>
/// request, it is be used to limit the number of actively running threads.<br>
- static ASTWorkerHandle Create(llvm::StringRef File, AsyncTaskRunner *Tasks,<br>
- Semaphore &Barrier, CppFile AST,<br>
- steady_clock::duration UpdateDebounce);<br>
+ static ASTWorkerHandle Create(PathRef FileName,<br>
+ TUScheduler::ASTCache &IdleASTs,<br>
+ AsyncTaskRunner *Tasks, Semaphore &Barrier,<br>
+ steady_clock::duration UpdateDebounce,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs,<br>
+ bool StorePreamblesInMemory,<br>
+ PreambleParsedCallback PreambleCallback);<br>
~ASTWorker();<br>
<br>
void update(ParseInputs Inputs, WantDiagnostics,<br>
@@ -92,6 +173,7 @@ public:<br>
<br>
std::shared_ptr<const PreambleData> getPossiblyStalePreamble() const;<br>
std::size_t getUsedBytes() const;<br>
+ bool isASTCached() const;<br>
<br>
private:<br>
// Must be called exactly once on processing thread. Will return after<br>
@@ -119,22 +201,30 @@ private:<br>
llvm::Optional<WantDiagnostics> UpdateType;<br>
};<br>
<br>
- const std::string File;<br>
+ /// Handles retention of ASTs.<br>
+ TUScheduler::ASTCache &IdleASTs;<br>
const bool RunSync;<br>
- // Time to wait after an update to see whether another update obsoletes it.<br>
+ /// Time to wait after an update to see whether another update obsoletes it.<br>
const steady_clock::duration UpdateDebounce;<br>
+ /// File that ASTWorker is reponsible for.<br>
+ const Path FileName;<br>
+ /// Whether to keep the built preambles in memory or on disk.<br>
+ const bool StorePreambleInMemory;<br>
+ /// Callback, passed to the preamble builder.<br>
+ const PreambleParsedCallback PreambleCallback;<br>
+ /// Helper class required to build the ASTs.<br>
+ const std::shared_ptr<PCHContainerOperations> PCHs;<br>
<br>
Semaphore &Barrier;<br>
- // AST and FileInputs are only accessed on the processing thread from run().<br>
- CppFile AST;<br>
- // Inputs, corresponding to the current state of AST.<br>
+ /// Inputs, corresponding to the current state of AST.<br>
ParseInputs FileInputs;<br>
- // Guards members used by both TUScheduler and the worker thread.<br>
+ /// CompilerInvocation used for FileInputs.<br>
+ std::unique_ptr<CompilerInvocation> Invocation;<br>
+ /// Size of the last AST<br>
+ /// Guards members used by both TUScheduler and the worker thread.<br>
mutable std::mutex Mutex;<br>
std::shared_ptr<const PreambleData> LastBuiltPreamble; /* GUARDED_BY(Mutex) */<br>
- // Result of getUsedBytes() after the last rebuild or read of AST.<br>
- std::size_t LastASTSize; /* GUARDED_BY(Mutex) */<br>
- // Set to true to signal run() to finish processing.<br>
+ /// Set to true to signal run() to finish processing.<br>
bool Done; /* GUARDED_BY(Mutex) */<br>
std::deque<Request> Requests; /* GUARDED_BY(Mutex) */<br>
mutable std::condition_variable RequestsCV;<br>
@@ -182,27 +272,37 @@ private:<br>
std::shared_ptr<ASTWorker> Worker;<br>
};<br>
<br>
-ASTWorkerHandle ASTWorker::Create(llvm::StringRef File, AsyncTaskRunner *Tasks,<br>
- Semaphore &Barrier, CppFile AST,<br>
- steady_clock::duration UpdateDebounce) {<br>
+ASTWorkerHandle ASTWorker::Create(PathRef FileName,<br>
+ TUScheduler::ASTCache &IdleASTs,<br>
+ AsyncTaskRunner *Tasks, Semaphore &Barrier,<br>
+ steady_clock::duration UpdateDebounce,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs,<br>
+ bool StorePreamblesInMemory,<br>
+ PreambleParsedCallback PreambleCallback) {<br>
std::shared_ptr<ASTWorker> Worker(new ASTWorker(<br>
- File, Barrier, std::move(AST), /*RunSync=*/!Tasks, UpdateDebounce));<br>
+ FileName, IdleASTs, Barrier, /*RunSync=*/!Tasks, UpdateDebounce,<br>
+ std::move(PCHs), StorePreamblesInMemory, std::move(PreambleCallback)));<br>
if (Tasks)<br>
- Tasks->runAsync("worker:" + llvm::sys::path::filename(File),<br>
+ Tasks->runAsync("worker:" + llvm::sys::path::filename(FileName),<br>
[Worker]() { Worker->run(); });<br>
<br>
return ASTWorkerHandle(std::move(Worker));<br>
}<br>
<br>
-ASTWorker::ASTWorker(llvm::StringRef File, Semaphore &Barrier, CppFile AST,<br>
- bool RunSync, steady_clock::duration UpdateDebounce)<br>
- : File(File), RunSync(RunSync), UpdateDebounce(UpdateDebounce),<br>
- Barrier(Barrier), AST(std::move(AST)), Done(false) {<br>
- if (RunSync)<br>
- return;<br>
-}<br>
+ASTWorker::ASTWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache,<br>
+ Semaphore &Barrier, bool RunSync,<br>
+ steady_clock::duration UpdateDebounce,<br>
+ std::shared_ptr<PCHContainerOperations> PCHs,<br>
+ bool StorePreamblesInMemory,<br>
+ PreambleParsedCallback PreambleCallback)<br>
+ : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),<br>
+ FileName(FileName), StorePreambleInMemory(StorePreamblesInMemory),<br>
+ PreambleCallback(std::move(PreambleCallback)), PCHs(std::move(PCHs)),<br>
+ Barrier(Barrier), Done(false) {}<br>
<br>
ASTWorker::~ASTWorker() {<br>
+ // Make sure we remove the cached AST, if any.<br>
+ IdleASTs.take(this);<br>
#ifndef NDEBUG<br>
std::lock_guard<std::mutex> Lock(Mutex);<br>
assert(Done && "handle was not destroyed");<br>
@@ -213,20 +313,41 @@ ASTWorker::~ASTWorker() {<br>
void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,<br>
UniqueFunction<void(std::vector<Diag>)> OnUpdated) {<br>
auto Task = [=](decltype(OnUpdated) OnUpdated) mutable {<br>
+ tooling::CompileCommand OldCommand = std::move(FileInputs.CompileCommand);<br>
FileInputs = Inputs;<br>
- auto Diags = AST.rebuild(std::move(Inputs));<br>
+ // Remove the old AST if it's still in cache.<br>
+ IdleASTs.take(this);<br>
+<br>
+ log("Updating file " + FileName + " with command [" +<br>
+ Inputs.CompileCommand.Directory + "] " +<br>
+ llvm::join(Inputs.CompileCommand.CommandLine, " "));<br>
+ // Rebuild the preamble and the AST.<br>
+ Invocation = buildCompilerInvocation(Inputs);<br>
+ if (!Invocation) {<br>
+ log("Could not build CompilerInvocation for file " + FileName);<br>
+ return;<br>
+ }<br>
<br>
+ std::shared_ptr<const PreambleData> NewPreamble = buildPreamble(<br>
+ FileName, *Invocation, getPossiblyStalePreamble(), OldCommand, Inputs,<br>
+ PCHs, StorePreambleInMemory, PreambleCallback);<br>
{<br>
std::lock_guard<std::mutex> Lock(Mutex);<br>
- if (AST.getPreamble())<br>
- LastBuiltPreamble = AST.getPreamble();<br>
- LastASTSize = AST.getUsedBytes();<br>
+ if (NewPreamble)<br>
+ LastBuiltPreamble = NewPreamble;<br>
}<br>
+ // Build the AST for diagnostics.<br>
+ llvm::Optional<ParsedAST> AST =<br>
+ buildAST(FileName, llvm::make_unique<CompilerInvocation>(*Invocation),<br>
+ Inputs, NewPreamble, PCHs);<br>
// We want to report the diagnostics even if this update was cancelled.<br>
// It seems more useful than making the clients wait indefinitely if they<br>
// spam us with updates.<br>
- if (Diags && WantDiags != WantDiagnostics::No)<br>
- OnUpdated(std::move(*Diags));<br>
+ if (WantDiags != WantDiagnostics::No && AST)<br>
+ OnUpdated(AST->getDiagnostics());<br>
+ // Stash the AST in the cache for further use.<br>
+ IdleASTs.put(this,<br>
+ AST ? llvm::make_unique<ParsedAST>(std::move(*AST)) : nullptr);<br>
};<br>
<br>
startTask("Update", Bind(Task, std::move(OnUpdated)), WantDiags);<br>
@@ -236,20 +357,26 @@ void ASTWorker::runWithAST(<br>
llvm::StringRef Name,<br>
UniqueFunction<void(llvm::Expected<InputsAndAST>)> Action) {<br>
auto Task = [=](decltype(Action) Action) {<br>
- ParsedAST *ActualAST = AST.getAST();<br>
- if (!ActualAST) {<br>
- Action(llvm::make_error<llvm::StringError>("invalid AST",<br>
- llvm::errc::invalid_argument));<br>
- return;<br>
+ llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this);<br>
+ if (!AST) {<br>
+ // Try rebuilding the AST.<br>
+ llvm::Optional<ParsedAST> NewAST =<br>
+ Invocation<br>
+ ? buildAST(FileName,<br>
+ llvm::make_unique<CompilerInvocation>(*Invocation),<br>
+ FileInputs, getPossiblyStalePreamble(), PCHs)<br>
+ : llvm::None;<br>
+ AST = NewAST ? llvm::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;<br>
}<br>
- Action(InputsAndAST{FileInputs, *ActualAST});<br>
-<br>
- // Size of the AST might have changed after reads too, e.g. if some decls<br>
- // were deserialized from preamble.<br>
- std::lock_guard<std::mutex> Lock(Mutex);<br>
- LastASTSize = ActualAST->getUsedBytes();<br>
+ // Make sure we put the AST back into the LRU cache.<br>
+ auto _ = llvm::make_scope_exit(<br>
+ [&AST, this]() { IdleASTs.put(this, std::move(*AST)); });<br>
+ // Run the user-provided action.<br>
+ if (!*AST)<br>
+ return Action(llvm::make_error<llvm::StringError>(<br>
+ "invalid AST", llvm::errc::invalid_argument));<br>
+ Action(InputsAndAST{FileInputs, **AST});<br>
};<br>
-<br>
startTask(Name, Bind(Task, std::move(Action)),<br>
/*UpdateType=*/llvm::None);<br>
}<br>
@@ -261,10 +388,17 @@ ASTWorker::getPossiblyStalePreamble() co<br>
}<br>
<br>
std::size_t ASTWorker::getUsedBytes() const {<br>
- std::lock_guard<std::mutex> Lock(Mutex);<br>
- return LastASTSize;<br>
+ // Note that we don't report the size of ASTs currently used for processing<br>
+ // the in-flight requests. We used this information for debugging purposes<br>
+ // only, so this should be fine.<br>
+ std::size_t Result = IdleASTs.getUsedBytes(this);<br>
+ if (auto Preamble = getPossiblyStalePreamble())<br>
+ Result += Preamble->Preamble.getSize();<br>
+ return Result;<br>
}<br>
<br>
+bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; }<br>
+<br>
void ASTWorker::stop() {<br>
{<br>
std::lock_guard<std::mutex> Lock(Mutex);<br>
@@ -278,7 +412,7 @@ void ASTWorker::startTask(llvm::StringRe<br>
llvm::Optional<WantDiagnostics> UpdateType) {<br>
if (RunSync) {<br>
assert(!Done && "running a task after stop()");<br>
- trace::Span Tracer(Name + ":" + llvm::sys::path::filename(File));<br>
+ trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName));<br>
Task();<br>
return;<br>
}<br>
@@ -415,10 +549,12 @@ struct TUScheduler::FileData {<br>
TUScheduler::TUScheduler(unsigned AsyncThreadsCount,<br>
bool StorePreamblesInMemory,<br>
PreambleParsedCallback PreambleCallback,<br>
- steady_clock::duration UpdateDebounce)<br>
+ std::chrono::steady_clock::duration UpdateDebounce,<br>
+ ASTRetentionPolicy RetentionPolicy)<br>
: StorePreamblesInMemory(StorePreamblesInMemory),<br>
PCHOps(std::make_shared<PCHContainerOperations>()),<br>
PreambleCallback(std::move(PreambleCallback)), Barrier(AsyncThreadsCount),<br>
+ IdleASTs(llvm::make_unique<ASTCache>(RetentionPolicy.MaxRetainedASTs)),<br>
UpdateDebounce(UpdateDebounce) {<br>
if (0 < AsyncThreadsCount) {<br>
PreambleTasks.emplace();<br>
@@ -454,9 +590,9 @@ void TUScheduler::update(PathRef File, P<br>
if (!FD) {<br>
// Create a new worker to process the AST-related tasks.<br>
ASTWorkerHandle Worker = ASTWorker::Create(<br>
- File, WorkerThreads ? WorkerThreads.getPointer() : nullptr, Barrier,<br>
- CppFile(File, StorePreamblesInMemory, PCHOps, PreambleCallback),<br>
- UpdateDebounce);<br>
+ File, *IdleASTs, WorkerThreads ? WorkerThreads.getPointer() : nullptr,<br>
+ Barrier, UpdateDebounce, PCHOps, StorePreamblesInMemory,<br>
+ PreambleCallback);<br>
FD = std::unique_ptr<FileData>(new FileData{<br>
Inputs.Contents, Inputs.CompileCommand, std::move(Worker)});<br>
} else {<br>
@@ -538,5 +674,15 @@ TUScheduler::getUsedBytesPerFile() const<br>
return Result;<br>
}<br>
<br>
+std::vector<Path> TUScheduler::getFilesWithCachedAST() const {<br>
+ std::vector<Path> Result;<br>
+ for (auto &&PathAndFile : Files) {<br>
+ if (!PathAndFile.second->Worker->isASTCached())<br>
+ continue;<br>
+ Result.push_back(PathAndFile.first());<br>
+ }<br>
+ return Result;<br>
+}<br>
+<br>
} // namespace clangd<br>
} // namespace clang<br>
<br>
Modified: clang-tools-extra/trunk/clangd/TUScheduler.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/TUScheduler.h?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/TUScheduler.h?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/clangd/TUScheduler.h (original)<br>
+++ clang-tools-extra/trunk/clangd/TUScheduler.h Fri Jun 1 03:08:43 2018<br>
@@ -42,6 +42,15 @@ enum class WantDiagnostics {<br>
/// within a bounded amount of time.<br>
};<br>
<br>
+/// Configuration of the AST retention policy. This only covers retention of<br>
+/// *idle* ASTs. If queue has operations requiring the AST, they might be<br>
+/// kept in memory.<br>
+struct ASTRetentionPolicy {<br>
+ /// Maximum number of ASTs to be retained in memory when there are no pending<br>
+ /// requests for them.<br>
+ unsigned MaxRetainedASTs = 3;<br>
+};<br>
+<br>
/// Handles running tasks for ClangdServer and managing the resources (e.g.,<br>
/// preambles and ASTs) for opened files.<br>
/// TUScheduler is not thread-safe, only one thread should be providing updates<br>
@@ -53,13 +62,19 @@ class TUScheduler {<br>
public:<br>
TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory,<br>
PreambleParsedCallback PreambleCallback,<br>
- std::chrono::steady_clock::duration UpdateDebounce);<br>
+ std::chrono::steady_clock::duration UpdateDebounce,<br>
+ ASTRetentionPolicy RetentionPolicy);<br>
~TUScheduler();<br>
<br>
/// Returns estimated memory usage for each of the currently open files.<br>
/// The order of results is unspecified.<br>
std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;<br>
<br>
+ /// Returns a list of files with ASTs currently stored in memory. This method<br>
+ /// is not very reliable and is only used for test. E.g., the results will not<br>
+ /// contain files that currently run something over their AST.<br>
+ std::vector<Path> getFilesWithCachedAST() const;<br>
+<br>
/// Schedule an update for \p File. Adds \p File to a list of tracked files if<br>
/// \p File was not part of it before.<br>
/// FIXME(ibiryukov): remove the callback from this function.<br>
@@ -99,11 +114,18 @@ private:<br>
/// This class stores per-file data in the Files map.<br>
struct FileData;<br>
<br>
+public:<br>
+ /// Responsible for retaining and rebuilding idle ASTs. An implementation is<br>
+ /// an LRU cache.<br>
+ class ASTCache;<br>
+<br>
+private:<br>
const bool StorePreamblesInMemory;<br>
const std::shared_ptr<PCHContainerOperations> PCHOps;<br>
const PreambleParsedCallback PreambleCallback;<br>
Semaphore Barrier;<br>
llvm::StringMap<std::unique_ptr<FileData>> Files;<br>
+ std::unique_ptr<ASTCache> IdleASTs;<br>
// None when running tasks synchronously and non-None when running tasks<br>
// asynchronously.<br>
llvm::Optional<AsyncTaskRunner> PreambleTasks;<br>
<br>
Modified: clang-tools-extra/trunk/test/clangd/trace.test<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/trace.test?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/trace.test?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/test/clangd/trace.test (original)<br>
+++ clang-tools-extra/trunk/test/clangd/trace.test Fri Jun 1 03:08:43 2018<br>
@@ -8,14 +8,14 @@<br>
# CHECK: "args": {<br>
# CHECK: "File": "{{.*(/|\\)}}foo.c"<br>
# CHECK: },<br>
-# CHECK: "name": "Preamble",<br>
+# CHECK: "name": "BuildPreamble",<br>
# CHECK: "ph": "X",<br>
# CHECK: }<br>
# CHECK: {<br>
# CHECK: "args": {<br>
# CHECK: "File": "{{.*(/|\\)}}foo.c"<br>
# CHECK: },<br>
-# CHECK: "name": "Build",<br>
+# CHECK: "name": "BuildAST",<br>
# CHECK: "ph": "X",<br>
# CHECK: }<br>
# CHECK: },<br>
<br>
Modified: clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp (original)<br>
+++ clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp Fri Jun 1 03:08:43 2018<br>
@@ -11,6 +11,7 @@<br>
#include "TestFS.h"<br>
#include "TestTU.h"<br>
#include "index/FileIndex.h"<br>
+#include "clang/Frontend/CompilerInvocation.h"<br>
#include "clang/Frontend/PCHContainerOperations.h"<br>
#include "clang/Lex/Preprocessor.h"<br>
#include "clang/Tooling/CompilationDatabase.h"<br>
@@ -208,18 +209,6 @@ vector<Ty> make_vector(Arg A) {}<br>
TEST(FileIndexTest, RebuildWithPreamble) {<br>
auto FooCpp = testPath("foo.cpp");<br>
auto FooH = testPath("foo.h");<br>
- FileIndex Index;<br>
- bool IndexUpdated = false;<br>
- CppFile File("foo.cpp", /*StorePreambleInMemory=*/true,<br>
- std::make_shared<PCHContainerOperations>(),<br>
- [&Index, &IndexUpdated](PathRef FilePath, ASTContext &Ctx,<br>
- std::shared_ptr<Preprocessor> PP) {<br>
- EXPECT_FALSE(IndexUpdated)<br>
- << "Expected only a single index update";<br>
- IndexUpdated = true;<br>
- Index.update(FilePath, &Ctx, std::move(PP));<br>
- });<br>
-<br>
// Preparse ParseInputs.<br>
ParseInputs PI;<br>
PI.CompileCommand.Directory = testRoot();<br>
@@ -243,7 +232,19 @@ TEST(FileIndexTest, RebuildWithPreamble)<br>
)cpp";<br>
<br>
// Rebuild the file.<br>
- File.rebuild(std::move(PI));<br>
+ auto CI = buildCompilerInvocation(PI);<br>
+<br>
+ FileIndex Index;<br>
+ bool IndexUpdated = false;<br>
+ buildPreamble(<br>
+ FooCpp, *CI, /*OldPreamble=*/nullptr, tooling::CompileCommand(), PI,<br>
+ std::make_shared<PCHContainerOperations>(), /*StoreInMemory=*/true,<br>
+ [&Index, &IndexUpdated](PathRef FilePath, ASTContext &Ctx,<br>
+ std::shared_ptr<Preprocessor> PP) {<br>
+ EXPECT_FALSE(IndexUpdated) << "Expected only a single index update";<br>
+ IndexUpdated = true;<br>
+ Index.update(FilePath, &Ctx, std::move(PP));<br>
+ });<br>
ASSERT_TRUE(IndexUpdated);<br>
<br>
// Check the index contains symbols from the preamble, but not from the main<br>
<br>
Modified: clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp?rev=333737&r1=333736&r2=333737&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp?rev=333737&r1=333736&r2=333737&view=diff</a><br>
==============================================================================<br>
--- clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp (original)<br>
+++ clang-tools-extra/trunk/unittests/clangd/TUSchedulerTests.cpp Fri Jun 1 03:08:43 2018<br>
@@ -18,8 +18,11 @@<br>
namespace clang {<br>
namespace clangd {<br>
<br>
+using ::testing::_;<br>
+using ::testing::AnyOf;<br>
using ::testing::Pair;<br>
using ::testing::Pointee;<br>
+using ::testing::UnorderedElementsAre;<br>
<br>
void ignoreUpdate(llvm::Optional<std::vector<Diag>>) {}<br>
void ignoreError(llvm::Error Err) {<br>
@@ -43,7 +46,8 @@ TEST_F(TUSchedulerTests, MissingFiles) {<br>
TUScheduler S(getDefaultAsyncThreadsCount(),<br>
/*StorePreamblesInMemory=*/true,<br>
/*PreambleParsedCallback=*/nullptr,<br>
- /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero());<br>
+ /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(),<br>
+ ASTRetentionPolicy());<br>
<br>
auto Added = testPath("added.cpp");<br>
Files[Added] = "";<br>
@@ -99,7 +103,8 @@ TEST_F(TUSchedulerTests, WantDiagnostics<br>
getDefaultAsyncThreadsCount(),<br>
/*StorePreamblesInMemory=*/true,<br>
/*PreambleParsedCallback=*/nullptr,<br>
- /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero());<br>
+ /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(),<br>
+ ASTRetentionPolicy());<br>
auto Path = testPath("foo.cpp");<br>
S.update(Path, getInputs(Path, ""), WantDiagnostics::Yes,<br>
[&](std::vector<Diag>) { Ready.wait(); });<br>
@@ -127,7 +132,8 @@ TEST_F(TUSchedulerTests, Debounce) {<br>
TUScheduler S(getDefaultAsyncThreadsCount(),<br>
/*StorePreamblesInMemory=*/true,<br>
/*PreambleParsedCallback=*/nullptr,<br>
- /*UpdateDebounce=*/std::chrono::seconds(1));<br>
+ /*UpdateDebounce=*/std::chrono::seconds(1),<br>
+ ASTRetentionPolicy());<br>
// FIXME: we could probably use timeouts lower than 1 second here.<br>
auto Path = testPath("foo.cpp");<br>
S.update(Path, getInputs(Path, "auto (debounced)"), WantDiagnostics::Auto,<br>
@@ -158,7 +164,8 @@ TEST_F(TUSchedulerTests, ManyUpdates) {<br>
TUScheduler S(getDefaultAsyncThreadsCount(),<br>
/*StorePreamblesInMemory=*/true,<br>
/*PreambleParsedCallback=*/nullptr,<br>
- /*UpdateDebounce=*/std::chrono::milliseconds(50));<br>
+ /*UpdateDebounce=*/std::chrono::milliseconds(50),<br>
+ ASTRetentionPolicy());<br>
<br>
std::vector<std::string> Files;<br>
for (int I = 0; I < FilesCount; ++I) {<br>
@@ -219,18 +226,18 @@ TEST_F(TUSchedulerTests, ManyUpdates) {<br>
<br>
{<br>
WithContextValue WithNonce(NonceKey, ++Nonce);<br>
- S.runWithPreamble(<br>
- "CheckPreamble", File,<br>
- [Inputs, Nonce, &Mut, &TotalPreambleReads](<br>
- llvm::Expected<InputsAndPreamble> Preamble) {<br>
- EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));<br>
-<br>
- ASSERT_TRUE((bool)Preamble);<br>
- EXPECT_EQ(Preamble->Contents, Inputs.Contents);<br>
-<br>
- std::lock_guard<std::mutex> Lock(Mut);<br>
- ++TotalPreambleReads;<br>
- });<br>
+ S.runWithPreamble("CheckPreamble", File,<br>
+ [Inputs, Nonce, &Mut, &TotalPreambleReads](<br>
+ llvm::Expected<InputsAndPreamble> Preamble) {<br>
+ EXPECT_THAT(Context::current().get(NonceKey),<br>
+ Pointee(Nonce));<br>
+<br>
+ ASSERT_TRUE((bool)Preamble);<br>
+ EXPECT_EQ(Preamble->Contents, Inputs.Contents);<br>
+<br>
+ std::lock_guard<std::mutex> Lock(Mut);<br>
+ ++TotalPreambleReads;<br>
+ });<br>
}<br>
}<br>
}<br>
@@ -242,5 +249,55 @@ TEST_F(TUSchedulerTests, ManyUpdates) {<br>
EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);<br>
}<br>
<br>
+TEST_F(TUSchedulerTests, EvictedAST) {<br>
+ ASTRetentionPolicy Policy;<br>
+ Policy.MaxRetainedASTs = 2;<br>
+ TUScheduler S(<br>
+ /*AsyncThreadsCount=*/1, /*StorePreambleInMemory=*/true,<br>
+ PreambleParsedCallback(),<br>
+ /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), Policy);<br>
+<br>
+ llvm::StringLiteral SourceContents = R"cpp(<br>
+ int* a;<br>
+ double* b = a;<br>
+ )cpp";<br>
+<br>
+ auto Foo = testPath("foo.cpp");<br>
+ auto Bar = testPath("bar.cpp");<br>
+ auto Baz = testPath("baz.cpp");<br>
+<br>
+ std::atomic<int> BuiltASTCounter;<br>
+ BuiltASTCounter = false;<br>
+ // Build one file in advance. We will not access it later, so it will be the<br>
+ // one that the cache will evict.<br>
+ S.update(Foo, getInputs(Foo, SourceContents), WantDiagnostics::Yes,<br>
+ [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; });<br>
+ ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1)));<br>
+ ASSERT_EQ(BuiltASTCounter.load(), 1);<br>
+<br>
+ // Build two more files. Since we can retain only 2 ASTs, these should be the<br>
+ // ones we see in the cache later.<br>
+ S.update(Bar, getInputs(Bar, SourceContents), WantDiagnostics::Yes,<br>
+ [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; });<br>
+ S.update(Baz, getInputs(Baz, SourceContents), WantDiagnostics::Yes,<br>
+ [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; });<br>
+ ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1)));<br>
+ ASSERT_EQ(BuiltASTCounter.load(), 3);<br>
+<br>
+ // Check only the last two ASTs are retained.<br>
+ ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));<br>
+<br>
+ // Access the old file again.<br>
+ S.update(Foo, getInputs(Foo, SourceContents), WantDiagnostics::Yes,<br>
+ [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; });<br>
+ ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1)));<br>
+ ASSERT_EQ(BuiltASTCounter.load(), 4);<br>
+<br>
+ // Check the AST for foo.cpp is retained now and one of the others got<br>
+ // evicted.<br>
+ EXPECT_THAT(S.getFilesWithCachedAST(),<br>
+ UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));<br>
+}<br>
+<br>
} // namespace clangd<br>
} // namespace clang<br>
<br>
<br>
_______________________________________________<br>
cfe-commits mailing list<br>
<a href="mailto:cfe-commits@lists.llvm.org" target="_blank">cfe-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits</a><br>
</blockquote></div><br clear="all"><div><br></div>-- <br><div dir="ltr" class="m_9100123605669742476gmail_signature" data-smartmail="gmail_signature"><div dir="ltr"><div><div dir="ltr"><div>Regards,</div><div>Ilya Biryukov</div></div></div></div></div>