[clang-tools-extra] a74d594 - [clangd] Introduce memory dumping to FileIndex, FileSymbols and BackgroundIndex
Kadir Cetinkaya via cfe-commits
cfe-commits at lists.llvm.org
Mon Oct 12 06:27:23 PDT 2020
Author: Kadir Cetinkaya
Date: 2020-10-12T15:25:29+02:00
New Revision: a74d594948611164f88a79ca0544721183a0b19c
URL: https://github.com/llvm/llvm-project/commit/a74d594948611164f88a79ca0544721183a0b19c
DIFF: https://github.com/llvm/llvm-project/commit/a74d594948611164f88a79ca0544721183a0b19c.diff
LOG: [clangd] Introduce memory dumping to FileIndex, FileSymbols and BackgroundIndex
File-granular information is considered details.
Depends on D88411
Differential Revision: https://reviews.llvm.org/D88414
Added:
Modified:
clang-tools-extra/clangd/index/Background.cpp
clang-tools-extra/clangd/index/Background.h
clang-tools-extra/clangd/index/FileIndex.cpp
clang-tools-extra/clangd/index/FileIndex.h
clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
clang-tools-extra/clangd/unittests/FileIndexTests.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp
index a1aafeaf31a9..4779cb8d4c23 100644
--- a/clang-tools-extra/clangd/index/Background.cpp
+++ b/clang-tools-extra/clangd/index/Background.cpp
@@ -16,6 +16,7 @@
#include "URI.h"
#include "index/BackgroundIndexLoader.h"
#include "index/FileIndex.h"
+#include "index/Index.h"
#include "index/IndexAction.h"
#include "index/MemIndex.h"
#include "index/Ref.h"
@@ -414,5 +415,10 @@ BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
return {TUsToIndex.begin(), TUsToIndex.end()};
}
+void BackgroundIndex::profile(MemoryTree &MT) const {
+ IndexedSymbols.profile(MT.child("symbols"));
+ // We don't want to mix memory used by index and symbols, so call base class.
+ MT.child("index").addUsage(SwapIndex::estimateMemoryUsage());
+}
} // namespace clangd
} // namespace clang
diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h
index 472603013a53..e8f9468889f2 100644
--- a/clang-tools-extra/clangd/index/Background.h
+++ b/clang-tools-extra/clangd/index/Background.h
@@ -16,9 +16,11 @@
#include "index/Index.h"
#include "index/Serialization.h"
#include "support/Context.h"
+#include "support/MemoryTree.h"
#include "support/Path.h"
#include "support/Threading.h"
#include "support/ThreadsafeFS.h"
+#include "support/Trace.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/Threading.h"
@@ -172,6 +174,8 @@ class BackgroundIndex : public SwapIndex {
return Queue.blockUntilIdleForTest(TimeoutSeconds);
}
+ void profile(MemoryTree &MT) const;
+
private:
/// Represents the state of a single file when indexing was performed.
struct ShardVersion {
diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp
index ad55b6ad7f5d..587c7eb78170 100644
--- a/clang-tools-extra/clangd/index/FileIndex.cpp
+++ b/clang-tools-extra/clangd/index/FileIndex.cpp
@@ -22,6 +22,7 @@
#include "index/SymbolOrigin.h"
#include "index/dex/Dex.h"
#include "support/Logger.h"
+#include "support/MemoryTree.h"
#include "support/Path.h"
#include "clang/AST/ASTContext.h"
#include "clang/Index/IndexingAction.h"
@@ -388,6 +389,25 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
llvm_unreachable("Unknown clangd::IndexType");
}
+void FileSymbols::profile(MemoryTree &MT) const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ for (const auto &SymSlab : SymbolsSnapshot) {
+ MT.detail(SymSlab.first())
+ .child("symbols")
+ .addUsage(SymSlab.second->bytes());
+ }
+ for (const auto &RefSlab : RefsSnapshot) {
+ MT.detail(RefSlab.first())
+ .child("references")
+ .addUsage(RefSlab.second.Slab->bytes());
+ }
+ for (const auto &RelSlab : RelationsSnapshot) {
+ MT.detail(RelSlab.first())
+ .child("relations")
+ .addUsage(RelSlab.second->bytes());
+ }
+}
+
FileIndex::FileIndex(bool UseDex, bool CollectMainFileRefs)
: MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
CollectMainFileRefs(CollectMainFileRefs),
@@ -457,5 +477,15 @@ void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
}
}
+void FileIndex::profile(MemoryTree &MT) const {
+ PreambleSymbols.profile(MT.child("preamble").child("symbols"));
+ MT.child("preamble")
+ .child("index")
+ .addUsage(PreambleIndex.estimateMemoryUsage());
+ MainFileSymbols.profile(MT.child("main_file").child("symbols"));
+ MT.child("main_file")
+ .child("index")
+ .addUsage(MainFileIndex.estimateMemoryUsage());
+}
} // namespace clangd
} // namespace clang
diff --git a/clang-tools-extra/clangd/index/FileIndex.h b/clang-tools-extra/clangd/index/FileIndex.h
index 127203c84c48..8ecae66373a5 100644
--- a/clang-tools-extra/clangd/index/FileIndex.h
+++ b/clang-tools-extra/clangd/index/FileIndex.h
@@ -24,6 +24,7 @@
#include "index/Relation.h"
#include "index/Serialization.h"
#include "index/Symbol.h"
+#include "support/MemoryTree.h"
#include "support/Path.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Tooling/CompilationDatabase.h"
@@ -87,6 +88,8 @@ class FileSymbols {
DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne,
size_t *Version = nullptr);
+ void profile(MemoryTree &MT) const;
+
private:
struct RefSlabAndCountReferences {
std::shared_ptr<RefSlab> Slab;
@@ -116,6 +119,8 @@ class FileIndex : public MergedIndex {
/// `indexMainDecls`.
void updateMain(PathRef Path, ParsedAST &AST);
+ void profile(MemoryTree &MT) const;
+
private:
bool UseDex; // FIXME: this should be always on.
bool CollectMainFileRefs;
diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
index adf39a915c1a..4f089519530a 100644
--- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -21,6 +21,7 @@ using ::testing::AllOf;
using ::testing::Contains;
using ::testing::ElementsAre;
using ::testing::Not;
+using ::testing::Pair;
using ::testing::UnorderedElementsAre;
namespace clang {
@@ -916,5 +917,18 @@ TEST(BackgroundQueueTest, Progress) {
EXPECT_EQ(S.LastIdle, 2000u);
}
+TEST(BackgroundIndex, Profile) {
+ MockFS FS;
+ MockCompilationDatabase CDB;
+ BackgroundIndex Idx(FS, CDB, [](llvm::StringRef) { return nullptr; },
+ /*Opts=*/{});
+
+ llvm::BumpPtrAllocator Alloc;
+ MemoryTree MT(&Alloc);
+ Idx.profile(MT);
+ ASSERT_THAT(MT.children(),
+ UnorderedElementsAre(Pair("symbols", _), Pair("index", _)));
+}
+
} // namespace clangd
} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
index c5bfbe132d37..2b20b7e7fef0 100644
--- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp
@@ -22,20 +22,25 @@
#include "index/Relation.h"
#include "index/Serialization.h"
#include "index/Symbol.h"
+#include "index/SymbolID.h"
#include "support/Threading.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/Utils.h"
#include "clang/Index/IndexSymbol.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Allocator.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <utility>
+#include <vector>
using ::testing::_;
using ::testing::AllOf;
using ::testing::Contains;
using ::testing::ElementsAre;
+using ::testing::Gt;
using ::testing::IsEmpty;
using ::testing::Pair;
using ::testing::UnorderedElementsAre;
@@ -88,6 +93,13 @@ std::unique_ptr<RefSlab> refSlab(const SymbolID &ID, const char *Path) {
return std::make_unique<RefSlab>(std::move(Slab).build());
}
+std::unique_ptr<RelationSlab> relSlab(llvm::ArrayRef<const Relation> Rels) {
+ RelationSlab::Builder RelBuilder;
+ for (auto &Rel : Rels)
+ RelBuilder.insert(Rel);
+ return std::make_unique<RelationSlab>(std::move(RelBuilder).build());
+}
+
TEST(FileSymbolsTest, UpdateAndGet) {
FileSymbols FS;
EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty());
@@ -643,6 +655,50 @@ TEST(FileShardedIndexTest, Sharding) {
EXPECT_TRUE(Shard->Cmd.hasValue());
}
}
+
+TEST(FileIndexTest, Profile) {
+ FileIndex FI;
+
+ auto FileName = testPath("foo.cpp");
+ auto AST = TestTU::withHeaderCode("int a;").build();
+ FI.updateMain(FileName, AST);
+ FI.updatePreamble(FileName, "v1", AST.getASTContext(),
+ AST.getPreprocessorPtr(), AST.getCanonicalIncludes());
+
+ llvm::BumpPtrAllocator Alloc;
+ MemoryTree MT(&Alloc);
+ FI.profile(MT);
+ ASSERT_THAT(MT.children(),
+ UnorderedElementsAre(Pair("preamble", _), Pair("main_file", _)));
+
+ ASSERT_THAT(MT.child("preamble").children(),
+ UnorderedElementsAre(Pair("index", _), Pair("symbols", _)));
+ ASSERT_THAT(MT.child("main_file").children(),
+ UnorderedElementsAre(Pair("index", _), Pair("symbols", _)));
+
+ ASSERT_THAT(MT.child("preamble").child("index").total(), Gt(0U));
+ ASSERT_THAT(MT.child("main_file").child("index").total(), Gt(0U));
+}
+
+TEST(FileSymbolsTest, Profile) {
+ FileSymbols FS;
+ FS.update("f1", numSlab(1, 2), nullptr, nullptr, false);
+ FS.update("f2", nullptr, refSlab(SymbolID("1"), "f1"), nullptr, false);
+ FS.update("f3", nullptr, nullptr,
+ relSlab({{SymbolID("1"), RelationKind::BaseOf, SymbolID("2")}}),
+ false);
+ llvm::BumpPtrAllocator Alloc;
+ MemoryTree MT(&Alloc);
+ FS.profile(MT);
+ ASSERT_THAT(MT.children(), UnorderedElementsAre(Pair("f1", _), Pair("f2", _),
+ Pair("f3", _)));
+ EXPECT_THAT(MT.child("f1").children(), ElementsAre(Pair("symbols", _)));
+ EXPECT_THAT(MT.child("f1").total(), Gt(0U));
+ EXPECT_THAT(MT.child("f2").children(), ElementsAre(Pair("references", _)));
+ EXPECT_THAT(MT.child("f2").total(), Gt(0U));
+ EXPECT_THAT(MT.child("f3").children(), ElementsAre(Pair("relations", _)));
+ EXPECT_THAT(MT.child("f3").total(), Gt(0U));
+}
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list