[PATCH] D124240: [clangd][NFC] Reduce memory usage while building dex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet created this revision.
kadircet added reviewers: sammccall, kbobyrev.
Herald added subscribers: usaxena95, arphaman.
Herald added a project: All.
kadircet requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay, ilya-biryukov.
Herald added a project: clang-tools-extra.

Posting lists were copied into the final inverted index rather than
being moved. Also we weren't shrinking the vectors. This patch should help with
both peak memory usage while building dex (preamble/background-index) and it's
idle memory usage (unclear if copy constructor of vector preserves the capacity
or not).


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D124240

Files:
  clang-tools-extra/clangd/index/dex/Dex.cpp


Index: clang-tools-extra/clangd/index/dex/Dex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/Dex.cpp
+++ clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Support/ScopedPrinter.h"
 #include <algorithm>
 #include <queue>
+#include <utility>
 
 namespace clang {
 namespace clangd {
@@ -76,23 +77,42 @@
   }
 
   // Assemble the final compressed posting lists for the added symbols.
-  llvm::DenseMap<Token, PostingList> build() {
+  llvm::DenseMap<Token, PostingList> build() && {
     llvm::DenseMap<Token, PostingList> Result(/*InitialReserve=*/
                                               TrigramDocs.size() +
                                               RestrictedCCDocs.size() +
                                               TypeDocs.size() +
                                               ScopeDocs.size() +
                                               ProximityDocs.size());
-    for (const auto &E : TrigramDocs)
-      Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()), E.second);
-    for (const auto &E : TypeDocs)
-      Result.try_emplace(Token(Token::Kind::Type, E.first()), E.second);
-    for (const auto &E : ScopeDocs)
-      Result.try_emplace(Token(Token::Kind::Scope, E.first()), E.second);
-    for (const auto &E : ProximityDocs)
-      Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()), E.second);
-    if (!RestrictedCCDocs.empty())
-      Result.try_emplace(RestrictedForCodeCompletion, RestrictedCCDocs);
+    for (auto &E : TrigramDocs) {
+      E.second.shrink_to_fit();
+      Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()),
+                         std::move(E.second));
+    }
+    TrigramDocs.clear();
+    for (auto &E : TypeDocs) {
+      E.second.shrink_to_fit();
+      Result.try_emplace(Token(Token::Kind::Type, E.first()),
+                         std::move(E.second));
+    }
+    TypeDocs.clear();
+    for (auto &E : ScopeDocs) {
+      E.second.shrink_to_fit();
+      Result.try_emplace(Token(Token::Kind::Scope, E.first()),
+                         std::move(E.second));
+    }
+    ScopeDocs.clear();
+    for (auto &E : ProximityDocs) {
+      E.second.shrink_to_fit();
+      Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()),
+                         std::move(E.second));
+    }
+    ProximityDocs.clear();
+    if (!RestrictedCCDocs.empty()) {
+      RestrictedCCDocs.shrink_to_fit();
+      Result.try_emplace(RestrictedForCodeCompletion,
+                         std::move(RestrictedCCDocs));
+    }
     return Result;
   }
 };
@@ -125,7 +145,7 @@
   IndexBuilder Builder;
   for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
     Builder.add(*Symbols[SymbolRank], SymbolRank);
-  InvertedIndex = Builder.build();
+  InvertedIndex = std::move(Builder).build();
 }
 
 std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D124240.424406.patch
Type: text/x-patch
Size: 2964 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20220422/6cbc7f7d/attachment.bin>