[clang-tools-extra] r340601 - [clangd] Log memory usage of DexIndex and MemIndex
Kirill Bobyrev via cfe-commits
cfe-commits at lists.llvm.org
Fri Aug 24 02:12:55 PDT 2018
Author: omtcyfz
Date: Fri Aug 24 02:12:54 2018
New Revision: 340601
URL: http://llvm.org/viewvc/llvm-project?rev=340601&view=rev
Log:
[clangd] Log memory usage of DexIndex and MemIndex
This patch prints information about built index size estimation to
verbose logs. This is useful for optimizing memory usage of DexIndex and
comparisons with MemIndex.
Reviewed by: sammccall
Differential Revision: https://reviews.llvm.org/D51154
Modified:
clang-tools-extra/trunk/clangd/index/FileIndex.cpp
clang-tools-extra/trunk/clangd/index/FileIndex.h
clang-tools-extra/trunk/clangd/index/Index.h
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.h
clang-tools-extra/trunk/clangd/index/Merge.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/FileIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Fri Aug 24 02:12:54 2018
@@ -118,5 +118,9 @@ void FileIndex::findOccurrences(
log("findOccurrences is not implemented.");
}
+size_t FileIndex::estimateMemoryUsage() const {
+ return Index.estimateMemoryUsage();
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/index/FileIndex.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.h?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/FileIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/FileIndex.h Fri Aug 24 02:12:54 2018
@@ -81,6 +81,9 @@ public:
void findOccurrences(const OccurrencesRequest &Req,
llvm::function_ref<void(const SymbolOccurrence &)>
Callback) const override;
+
+ size_t estimateMemoryUsage() const override;
+
private:
FileSymbols FSymbols;
MemIndex Index;
Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Fri Aug 24 02:12:54 2018
@@ -385,6 +385,12 @@ public:
virtual void findOccurrences(
const OccurrencesRequest &Req,
llvm::function_ref<void(const SymbolOccurrence &)> Callback) const = 0;
+
+ /// Returns estimated size of index (in bytes).
+ // FIXME(kbobyrev): Currently, this only returns the size of index itself
+ // excluding the size of actual symbol slab index refers to. We should include
+ // both.
+ virtual size_t estimateMemoryUsage() const = 0;
};
} // namespace clangd
Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Fri Aug 24 02:12:54 2018
@@ -26,6 +26,9 @@ void MemIndex::build(std::shared_ptr<std
Index = std::move(TempIndex);
Symbols = std::move(Syms); // Relase old symbols.
}
+
+ vlog("Built MemIndex with estimated memory usage {0} bytes.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@ getSymbolsFromSlab(SymbolSlab Slab) {
&Snap->Pointers);
}
+size_t MemIndex::estimateMemoryUsage() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ return Index.getMemorySize();
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/MemIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.h Fri Aug 24 02:12:54 2018
@@ -39,7 +39,10 @@ public:
llvm::function_ref<void(const SymbolOccurrence &)>
Callback) const override;
+ size_t estimateMemoryUsage() const override;
+
private:
+
std::shared_ptr<std::vector<const Symbol *>> Symbols;
// Index is a set of symbols that are deduplicated by symbol IDs.
// FIXME: build smarter index structure.
Modified: clang-tools-extra/trunk/clangd/index/Merge.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Merge.cpp?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/Merge.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Merge.cpp Fri Aug 24 02:12:54 2018
@@ -84,6 +84,10 @@ class MergedIndex : public SymbolIndex {
log("findOccurrences is not implemented.");
}
+ size_t estimateMemoryUsage() const override {
+ return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage();
+ }
+
private:
const SymbolIndex *Dynamic, *Static;
};
Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Fri Aug 24 02:12:54 2018
@@ -67,6 +67,9 @@ void DexIndex::build(std::shared_ptr<std
InvertedIndex = std::move(TempInvertedIndex);
SymbolQuality = std::move(TempSymbolQuality);
}
+
+ vlog("Built DexIndex with estimated memory usage {0} bytes.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
@@ -171,6 +174,20 @@ void DexIndex::findOccurrences(
log("findOccurrences is not implemented.");
}
+size_t DexIndex::estimateMemoryUsage() const {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ size_t Bytes =
+ LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+ Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
+ Bytes += InvertedIndex.size() * sizeof(Token);
+
+ for (const auto &P : InvertedIndex) {
+ Bytes += P.second.size() * sizeof(DocID);
+ }
+ return Bytes;
+}
+
} // namespace dex
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.h?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.h Fri Aug 24 02:12:54 2018
@@ -57,7 +57,10 @@ public:
llvm::function_ref<void(const SymbolOccurrence &)>
Callback) const override;
+ size_t estimateMemoryUsage() const override;
+
private:
+
mutable std::mutex Mutex;
std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/;
Modified: clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp?rev=340601&r1=340600&r2=340601&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp Fri Aug 24 02:12:54 2018
@@ -923,6 +923,10 @@ public:
llvm::function_ref<void(const SymbolOccurrence &)>
Callback) const override {}
+ // This is incorrect, but IndexRequestCollector is not an actual index and it
+ // isn't used in production code.
+ size_t estimateMemoryUsage() const override { return 0; }
+
const std::vector<FuzzyFindRequest> allRequests() const { return Requests; }
private:
More information about the cfe-commits
mailing list