[clang-tools-extra] f407c9e - [clangd] Export preamble AST and serialized size as metrics

[Sam McCall via cfe-commits]

Author: Sam McCall
Date: 2022-04-13T14:43:06+02:00
New Revision: f407c9ed1057c4093cd09c0f8cee16f68c63157e

URL: https://github.com/llvm/llvm-project/commit/f407c9ed1057c4093cd09c0f8cee16f68c63157e
DIFF: https://github.com/llvm/llvm-project/commit/f407c9ed1057c4093cd09c0f8cee16f68c63157e.diff

LOG: [clangd] Export preamble AST and serialized size as metrics

Differential Revision: https://reviews.llvm.org/D123672

Added: 
    

Modified: 
    clang-tools-extra/clangd/Preamble.cpp
    clang-tools-extra/clangd/Preamble.h
    clang-tools-extra/clangd/TUScheduler.cpp

Removed: 
    


################################################################################
diff  --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index b3a3aec985ba0..0cc5b65a94d64 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -64,8 +64,9 @@ bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
 
 class CppFilePreambleCallbacks : public PreambleCallbacks {
 public:
-  CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
-      : File(File), ParsedCallback(ParsedCallback) {}
+  CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback,
+                           PreambleBuildStats *Stats)
+      : File(File), ParsedCallback(ParsedCallback), Stats(Stats) {}
 
   IncludeStructure takeIncludes() { return std::move(Includes); }
 
@@ -88,6 +89,25 @@ class CppFilePreambleCallbacks : public PreambleCallbacks {
     IsMainFileIncludeGuarded =
         CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
             MainFE);
+
+    if (Stats) {
+      const ASTContext &AST = CI.getASTContext();
+      Stats->BuildSize = AST.getASTAllocatedMemory();
+      Stats->BuildSize += AST.getSideTableAllocatedMemory();
+      Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
+      Stats->BuildSize += AST.Selectors.getTotalMemory();
+
+      Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
+      Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
+      Stats->BuildSize +=
+          AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
+
+      const Preprocessor &PP = CI.getPreprocessor();
+      Stats->BuildSize += PP.getTotalMemory();
+      if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
+        Stats->BuildSize += PRec->getTotalMemory();
+      Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
+    }
   }
 
   void BeforeExecute(CompilerInstance &CI) override {
@@ -135,6 +155,7 @@ class CppFilePreambleCallbacks : public PreambleCallbacks {
   std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
   const clang::LangOptions *LangOpts = nullptr;
   const SourceManager *SourceMgr = nullptr;
+  PreambleBuildStats *Stats;
 };
 
 // Represents directives other than includes, where basic textual information is
@@ -456,7 +477,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   // to read back. We rely on dynamic index for the comments instead.
   CI.getPreprocessorOpts().WriteCommentListToPCH = false;
 
-  CppFilePreambleCallbacks CapturedInfo(FileName, PreambleCallback);
+  CppFilePreambleCallbacks CapturedInfo(FileName, PreambleCallback, Stats);
   auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
   llvm::SmallString<32> AbsFileName(FileName);
   VFS->makeAbsolute(AbsFileName);
@@ -476,6 +497,12 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   // bodies.
   CI.getFrontendOpts().SkipFunctionBodies = false;
 
+  if (Stats != nullptr) {
+    Stats->TotalBuildTime = PreambleTimer.getTime();
+    Stats->FileSystemTime = TimedFS->getTime();
+    Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
+  }
+
   if (BuiltPreamble) {
     vlog("Built preamble of size {0} for file {1} version {2} in {3} seconds",
          BuiltPreamble->getSize(), FileName, Inputs.Version,
@@ -491,10 +518,6 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
     Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes();
     Result->StatCache = std::move(StatCache);
     Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
-    if (Stats != nullptr) {
-      Stats->TotalBuildTime = PreambleTimer.getTime();
-      Stats->FileSystemTime = TimedFS->getTime();
-    }
     return Result;
   }
 

diff  --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h
index 33a308484c189..f108dd4fd69fc 100644
--- a/clang-tools-extra/clangd/Preamble.h
+++ b/clang-tools-extra/clangd/Preamble.h
@@ -83,6 +83,14 @@ struct PreambleBuildStats {
   double TotalBuildTime;
   /// Time spent in filesystem operations during the build, in seconds.
   double FileSystemTime;
+
+  /// Estimate of the memory used while building the preamble.
+  /// This memory has been released when buildPreamble returns.
+  /// For example, this includes the size of the in-memory AST (ASTContext).
+  size_t BuildSize;
+  /// The serialized size of the preamble.
+  /// This storage is needed while the preamble is used (but may be on disk).
+  size_t SerializedSize;
 };
 
 /// Build a preamble for the new inputs unless an old one can be reused.

diff  --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index 928ab56d21ae3..7c5a52f6ea443 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -110,6 +110,11 @@ constexpr trace::Metric
 constexpr trace::Metric PreambleBuildFilesystemLatencyRatio(
     "preamble_fs_latency_ratio", trace::Metric::Distribution, "build_type");
 
+constexpr trace::Metric PreambleBuildSize("preamble_build_size",
+                                          trace::Metric::Distribution);
+constexpr trace::Metric PreambleSerializedSize("preamble_serialized_size",
+                                               trace::Metric::Distribution);
+
 void reportPreambleBuild(const PreambleBuildStats &Stats,
                          bool IsFirstPreamble) {
   auto RecordWithLabel = [&Stats](llvm::StringRef Label) {
@@ -122,6 +127,9 @@ void reportPreambleBuild(const PreambleBuildStats &Stats,
   static llvm::once_flag OnceFlag;
   llvm::call_once(OnceFlag, [&] { RecordWithLabel("first_build"); });
   RecordWithLabel(IsFirstPreamble ? "first_build_for_file" : "rebuild");
+
+  PreambleBuildSize.record(Stats.BuildSize);
+  PreambleSerializedSize.record(Stats.SerializedSize);
 }
 
 class ASTWorker;