r329784 - [Tooling] Optimize memory usage in InMemoryToolResults.

[Haojian Wu via cfe-commits]

Author: hokein
Date: Wed Apr 11 01:13:07 2018
New Revision: 329784

URL: http://llvm.org/viewvc/llvm-project?rev=329784&view=rev
Log:
[Tooling] Optimize memory usage in InMemoryToolResults.

Avoid storing duplicated "std::string"s.

clangd's global-symbol-builder takes 20+GB memory running across LLVM
repository. With this patch, the used memory is ~10GB (running on 48
threads, most of meory are AST-related).

Subscribers: klimek, cfe-commits

Differential Revision: https://reviews.llvm.org/D45479

Modified:
    cfe/trunk/include/clang/Tooling/Execution.h
    cfe/trunk/lib/Tooling/AllTUsExecution.cpp
    cfe/trunk/lib/Tooling/Execution.cpp

Modified: cfe/trunk/include/clang/Tooling/Execution.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Execution.h?rev=329784&r1=329783&r2=329784&view=diff
==============================================================================
--- cfe/trunk/include/clang/Tooling/Execution.h (original)
+++ cfe/trunk/include/clang/Tooling/Execution.h Wed Apr 11 01:13:07 2018
@@ -32,6 +32,7 @@
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Registry.h"
+#include "llvm/Support/StringSaver.h"
 
 namespace clang {
 namespace tooling {
@@ -45,20 +46,30 @@ class ToolResults {
 public:
   virtual ~ToolResults() = default;
   virtual void addResult(StringRef Key, StringRef Value) = 0;
-  virtual std::vector<std::pair<std::string, std::string>> AllKVResults() = 0;
+  virtual std::vector<std::pair<llvm::StringRef, llvm::StringRef>>
+  AllKVResults() = 0;
   virtual void forEachResult(
       llvm::function_ref<void(StringRef Key, StringRef Value)> Callback) = 0;
 };
 
+/// \brief Stores the key-value results in memory. It maintains the lifetime of
+/// the result. Clang tools using this class are expected to generate a small
+/// set of different results, or a large set of duplicated results.
 class InMemoryToolResults : public ToolResults {
 public:
+  InMemoryToolResults() : StringsPool(Arena) {}
   void addResult(StringRef Key, StringRef Value) override;
-  std::vector<std::pair<std::string, std::string>> AllKVResults() override;
+  std::vector<std::pair<llvm::StringRef, llvm::StringRef>>
+  AllKVResults() override;
   void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)>
                          Callback) override;
 
 private:
-  std::vector<std::pair<std::string, std::string>> KVResults;
+  llvm::BumpPtrAllocator Arena;
+  llvm::StringSaver StringsPool;
+  llvm::DenseSet<llvm::StringRef> Strings;
+
+  std::vector<std::pair<llvm::StringRef, llvm::StringRef>> KVResults;
 };
 
 /// \brief The context of an execution, including the information about

Modified: cfe/trunk/lib/Tooling/AllTUsExecution.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/AllTUsExecution.cpp?rev=329784&r1=329783&r2=329784&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/AllTUsExecution.cpp (original)
+++ cfe/trunk/lib/Tooling/AllTUsExecution.cpp Wed Apr 11 01:13:07 2018
@@ -36,7 +36,8 @@ public:
     Results.addResult(Key, Value);
   }
 
-  std::vector<std::pair<std::string, std::string>> AllKVResults() override {
+  std::vector<std::pair<llvm::StringRef, llvm::StringRef>>
+  AllKVResults() override {
     return Results.AllKVResults();
   }
 

Modified: cfe/trunk/lib/Tooling/Execution.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Execution.cpp?rev=329784&r1=329783&r2=329784&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/Execution.cpp (original)
+++ cfe/trunk/lib/Tooling/Execution.cpp Wed Apr 11 01:13:07 2018
@@ -21,10 +21,19 @@ static llvm::cl::opt<std::string>
                  llvm::cl::init("standalone"));
 
 void InMemoryToolResults::addResult(StringRef Key, StringRef Value) {
-  KVResults.push_back({Key.str(), Value.str()});
+  auto Intern = [&](StringRef &V) {
+    auto R = Strings.insert(V);
+    if (R.second) { // A new entry, create a new string copy.
+      *R.first = StringsPool.save(V);
+    }
+    V = *R.first;
+  };
+  Intern(Key);
+  Intern(Value);
+  KVResults.push_back({Key, Value});
 }
 
-std::vector<std::pair<std::string, std::string>>
+std::vector<std::pair<llvm::StringRef, llvm::StringRef>>
 InMemoryToolResults::AllKVResults() {
   return KVResults;
 }