[PATCH] D51154: [clangd] Log memory usage of DexIndex and MemIndex

[Kirill Bobyrev via Phabricator via cfe-commits]

kbobyrev created this revision.
kbobyrev added reviewers: ioeric, ilya-biryukov, sammccall.
kbobyrev added a project: clang-tools-extra.
Herald added subscribers: kadircet, arphaman, jkorous, MaskRay.

This patch prints information about built index size estimation to verbose logs. This is useful for optimizing memory usage of DexIndex and comparisons with MemIndex.


https://reviews.llvm.org/D51154

Files:
  clang-tools-extra/clangd/index/MemIndex.cpp
  clang-tools-extra/clangd/index/MemIndex.h
  clang-tools-extra/clangd/index/dex/DexIndex.cpp
  clang-tools-extra/clangd/index/dex/DexIndex.h


Index: clang-tools-extra/clangd/index/dex/DexIndex.h
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.h
+++ clang-tools-extra/clangd/index/dex/DexIndex.h
@@ -58,6 +58,9 @@
                            Callback) const override;
 
 private:
+  /// Returns estimate size of the index in megabytes.
+  size_t estimateMemoryUsage();
+
   mutable std::mutex Mutex;
 
   std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/;
Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.cpp
+++ clang-tools-extra/clangd/index/dex/DexIndex.cpp
@@ -67,6 +67,9 @@
     InvertedIndex = std::move(TempInvertedIndex);
     SymbolQuality = std::move(TempSymbolQuality);
   }
+
+  vlog("Built DexIndex with estimated memory usage {0} MB.",
+       estimateMemoryUsage());
 }
 
 std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
@@ -171,6 +174,20 @@
   log("findOccurrences is not implemented.");
 }
 
+size_t DexIndex::estimateMemoryUsage() {
+  size_t Bytes = LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+  Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
+  Bytes += InvertedIndex.size() * sizeof(Token);
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    for (const auto &P : InvertedIndex) {
+      Bytes += P.second.size() * sizeof(DocID);
+    }
+  }
+  return Bytes / (1000 * 1000);
+}
+
 } // namespace dex
 } // namespace clangd
 } // namespace clang
Index: clang-tools-extra/clangd/index/MemIndex.h
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.h
+++ clang-tools-extra/clangd/index/MemIndex.h
@@ -40,6 +40,9 @@
                            Callback) const override;
 
 private:
+  /// Returns estimate size of the index in megabytes.
+  size_t estimateMemoryUsage();
+
   std::shared_ptr<std::vector<const Symbol *>> Symbols;
   // Index is a set of symbols that are deduplicated by symbol IDs.
   // FIXME: build smarter index structure.
Index: clang-tools-extra/clangd/index/MemIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.cpp
+++ clang-tools-extra/clangd/index/MemIndex.cpp
@@ -26,6 +26,9 @@
     Index = std::move(TempIndex);
     Symbols = std::move(Syms); // Relase old symbols.
   }
+
+  vlog("Built MemIndex with estimated memory usage {0} MB.",
+       estimateMemoryUsage());
 }
 
 std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@
                                                       &Snap->Pointers);
 }
 
+size_t MemIndex::estimateMemoryUsage() {
+  size_t Bytes = Index.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+  return Bytes / (1000 * 1000);
+}
+
 } // namespace clangd
 } // namespace clang


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D51154.162141.patch
Type: text/x-patch
Size: 2959 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20180823/4bec3b5b/attachment.bin>