[clang-tools-extra] r323425 - [clangd] Provide a helper to report estimated memory usage per-file
Ilya Biryukov via cfe-commits
cfe-commits at lists.llvm.org
Thu Jan 25 06:32:21 PST 2018
Author: ibiryukov
Date: Thu Jan 25 06:32:21 2018
New Revision: 323425
URL: http://llvm.org/viewvc/llvm-project?rev=323425&view=rev
Log:
[clangd] Provide a helper to report estimated memory usage per-file
Reviewers: sammccall, ioeric, hokein
Reviewed By: ioeric
Subscribers: klimek, cfe-commits, jkorous-apple
Differential Revision: https://reviews.llvm.org/D42480
Modified:
clang-tools-extra/trunk/clangd/ClangdServer.cpp
clang-tools-extra/trunk/clangd/ClangdServer.h
clang-tools-extra/trunk/clangd/ClangdUnit.cpp
clang-tools-extra/trunk/clangd/ClangdUnit.h
clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp
clang-tools-extra/trunk/clangd/ClangdUnitStore.h
clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp
Modified: clang-tools-extra/trunk/clangd/ClangdServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.cpp Thu Jan 25 06:32:21 2018
@@ -635,3 +635,8 @@ void ClangdServer::onFileEvent(const Did
// FIXME: Do nothing for now. This will be used for indexing and potentially
// invalidating other caches.
}
+
+std::vector<std::pair<Path, std::size_t>>
+ClangdServer::getUsedBytesPerFile() const {
+ return Units.getUsedBytesPerFile();
+}
Modified: clang-tools-extra/trunk/clangd/ClangdServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.h?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.h Thu Jan 25 06:32:21 2018
@@ -322,6 +322,15 @@ public:
/// Called when an event occurs for a watched file in the workspace.
void onFileEvent(const DidChangeWatchedFilesParams &Params);
+ /// Returns estimated memory usage for each of the currently open files.
+ /// The order of results is unspecified.
+ /// Overall memory usage of clangd may be significantly more than reported
+ /// here, as this metric does not account (at least) for:
+ /// - memory occupied by static and dynamic index,
+ /// - memory required for in-flight requests,
+ /// FIXME: those metrics might be useful too, we should add them.
+ std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
+
private:
/// FIXME: This stats several files to find a .clang-format file. I/O can be
/// slow. Think of a way to cache this.
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Thu Jan 25 06:32:21 2018
@@ -35,6 +35,10 @@ using namespace clang;
namespace {
+template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
+ return Vec.capacity() * sizeof(T);
+}
+
class DeclTrackingASTConsumer : public ASTConsumer {
public:
DeclTrackingASTConsumer(std::vector<const Decl *> &TopLevelDecls)
@@ -332,6 +336,14 @@ const std::vector<DiagWithFixIts> &Parse
return Diags;
}
+std::size_t ParsedAST::getUsedBytes() const {
+ auto &AST = getASTContext();
+ // FIXME(ibiryukov): we do not account for the dynamically allocated part of
+ // SmallVector<FixIt> inside each Diag.
+ return AST.getASTAllocatedMemory() + AST.getSideTableAllocatedMemory() +
+ ::getUsedBytes(TopLevelDecls) + ::getUsedBytes(Diags);
+}
+
PreambleData::PreambleData(PrecompiledPreamble Preamble,
std::vector<serialization::DeclID> TopLevelDeclIDs,
std::vector<DiagWithFixIts> Diags)
@@ -370,7 +382,8 @@ CppFile::CppFile(PathRef FileName, bool
std::shared_ptr<PCHContainerOperations> PCHs,
ASTParsedCallback ASTCallback)
: FileName(FileName), StorePreamblesInMemory(StorePreamblesInMemory),
- RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)),
+ RebuildCounter(0), RebuildInProgress(false), ASTMemUsage(0),
+ PreambleMemUsage(0), PCHs(std::move(PCHs)),
ASTCallback(std::move(ASTCallback)) {
// FIXME(ibiryukov): we should pass a proper Context here.
log(Context::empty(), "Created CppFile for " + FileName);
@@ -419,7 +432,9 @@ UniqueFunction<void()> CppFile::deferCan
return;
// Set empty results for Promises.
+ That->PreambleMemUsage = 0;
That->PreamblePromise.set_value(nullptr);
+ That->ASTMemUsage = 0;
That->ASTPromise.set_value(std::make_shared<ParsedASTWrapper>(llvm::None));
};
}
@@ -573,6 +588,8 @@ CppFile::deferRebuild(ParseInputs &&Inpu
That->LatestAvailablePreamble = NewPreamble;
if (RequestRebuildCounter != That->RebuildCounter)
return llvm::None; // Our rebuild request was cancelled, do nothing.
+ That->PreambleMemUsage =
+ NewPreamble ? NewPreamble->Preamble.getSize() : 0;
That->PreamblePromise.set_value(NewPreamble);
} // unlock Mutex
@@ -609,6 +626,7 @@ CppFile::deferRebuild(ParseInputs &&Inpu
return Diagnostics; // Our rebuild request was cancelled, don't set
// ASTPromise.
+ That->ASTMemUsage = NewAST ? NewAST->getUsedBytes() : 0;
That->ASTPromise.set_value(
std::make_shared<ParsedASTWrapper>(std::move(NewAST)));
} // unlock Mutex
@@ -635,6 +653,14 @@ std::shared_future<std::shared_ptr<Parse
return ASTFuture;
}
+std::size_t CppFile::getUsedBytes() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ // FIXME: We should not store extra size fields. When we store AST and
+ // Preamble directly, not inside futures, we could compute the sizes from the
+ // stored AST and the preamble in this function directly.
+ return ASTMemUsage + PreambleMemUsage;
+}
+
CppFile::RebuildGuard::RebuildGuard(CppFile &File,
unsigned RequestRebuildCounter)
: File(File), RequestRebuildCounter(RequestRebuildCounter) {
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.h Thu Jan 25 06:32:21 2018
@@ -96,6 +96,10 @@ public:
const std::vector<DiagWithFixIts> &getDiagnostics() const;
+ /// Returns the esitmated size of the AST and the accessory structures, in
+ /// bytes. Does not include the size of the preamble.
+ std::size_t getUsedBytes() const;
+
private:
ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
@@ -216,6 +220,10 @@ public:
/// always be non-null.
std::shared_future<std::shared_ptr<ParsedASTWrapper>> getAST() const;
+ /// Returns an estimated size, in bytes, currently occupied by the AST and the
+ /// Preamble.
+ std::size_t getUsedBytes() const;
+
private:
/// A helper guard that manages the state of CppFile during rebuild.
class RebuildGuard {
@@ -244,6 +252,11 @@ private:
/// Condition variable to indicate changes to RebuildInProgress.
std::condition_variable RebuildCond;
+ /// Size of the last built AST, in bytes.
+ std::size_t ASTMemUsage;
+ /// Size of the last build Preamble, in bytes.
+ std::size_t PreambleMemUsage;
+
/// Promise and future for the latests AST. Fulfilled during rebuild.
/// We use std::shared_ptr here because MVSC fails to compile non-copyable
/// classes as template arguments of promise/future.
Modified: clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp Thu Jan 25 06:32:21 2018
@@ -25,3 +25,13 @@ std::shared_ptr<CppFile> CppFileCollecti
OpenedFiles.erase(It);
return Result;
}
+std::vector<std::pair<Path, std::size_t>>
+CppFileCollection::getUsedBytesPerFile() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ std::vector<std::pair<Path, std::size_t>> Result;
+ Result.reserve(OpenedFiles.size());
+ for (auto &&PathAndFile : OpenedFiles)
+ Result.push_back(
+ {PathAndFile.first().str(), PathAndFile.second->getUsedBytes()});
+ return Result;
+}
Modified: clang-tools-extra/trunk/clangd/ClangdUnitStore.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnitStore.h?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnitStore.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnitStore.h Thu Jan 25 06:32:21 2018
@@ -44,7 +44,7 @@ public:
return It->second;
}
- std::shared_ptr<CppFile> getFile(PathRef File) {
+ std::shared_ptr<CppFile> getFile(PathRef File) const {
std::lock_guard<std::mutex> Lock(Mutex);
auto It = OpenedFiles.find(File);
@@ -57,8 +57,11 @@ public:
/// returns it.
std::shared_ptr<CppFile> removeIfPresent(PathRef File);
+ /// Gets used memory for each of the stored files.
+ std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
+
private:
- std::mutex Mutex;
+ mutable std::mutex Mutex;
llvm::StringMap<std::shared_ptr<CppFile>> OpenedFiles;
ASTParsedCallback ASTCallback;
};
Modified: clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp?rev=323425&r1=323424&r2=323425&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp Thu Jan 25 06:32:21 2018
@@ -17,6 +17,7 @@
#include "llvm/Support/Errc.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Regex.h"
+#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <chrono>
@@ -28,6 +29,14 @@
namespace clang {
namespace clangd {
+
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::Gt;
+using ::testing::IsEmpty;
+using ::testing::Pair;
+using ::testing::UnorderedElementsAre;
+
namespace {
// Don't wait for async ops in clangd test more than that to avoid blocking
@@ -416,6 +425,42 @@ struct bar { T x; };
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
}
+TEST_F(ClangdVFSTest, MemoryUsage) {
+ MockFSProvider FS;
+ ErrorCheckingDiagConsumer DiagConsumer;
+ MockCompilationDatabase CDB;
+ ClangdServer Server(CDB, DiagConsumer, FS,
+ /*AsyncThreadsCount=*/0,
+ /*StorePreamblesInMemory=*/true);
+
+ // No need to sync reparses, because reparses are performed on the calling
+ // thread.
+ Path FooCpp = getVirtualTestFilePath("foo.cpp").str();
+ const auto SourceContents = R"cpp(
+struct Something {
+ int method();
+};
+)cpp";
+ Path BarCpp = getVirtualTestFilePath("bar.cpp").str();
+
+ FS.Files[FooCpp] = "";
+ FS.Files[BarCpp] = "";
+
+ EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty());
+
+ Server.addDocument(Context::empty(), FooCpp, SourceContents);
+ Server.addDocument(Context::empty(), BarCpp, SourceContents);
+
+ EXPECT_THAT(Server.getUsedBytesPerFile(),
+ UnorderedElementsAre(Pair(FooCpp, Gt(0u)), Pair(BarCpp, Gt(0u))));
+
+ Server.removeDocument(Context::empty(), FooCpp);
+ EXPECT_THAT(Server.getUsedBytesPerFile(), ElementsAre(Pair(BarCpp, Gt(0u))));
+
+ Server.removeDocument(Context::empty(), BarCpp);
+ EXPECT_THAT(Server.getUsedBytesPerFile(), IsEmpty());
+}
+
class ClangdThreadingTest : public ClangdVFSTest {};
TEST_F(ClangdThreadingTest, StressTest) {
More information about the cfe-commits
mailing list