[llvm-branch-commits] [clang-tools-extra] [clang-doc] Merge data into persistent memory (PR #190056)
Paul Kirth via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Apr 2 22:30:14 PDT 2026
https://github.com/ilovepi updated https://github.com/llvm/llvm-project/pull/190056
>From 6c2a519041520732d2d621efeed8008bf8affd96 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth at google.com>
Date: Sun, 29 Mar 2026 20:39:14 +0000
Subject: [PATCH] [clang-doc] Merge data into persistent memory
We have a need for persistent memory for the final info. Since each
group processes a single USR at a time, every USR is only ever processed by
a single thread from the thread pool. This means that we can keep per
thread persistent storage for all the info. There is significant
duplicated data between all the serialized records, so we can just merge
the final/unique items into the persistent arena, and clear out the
scratch/transient arena as we process each record in the bitcode.
The patch adds some APIs to help with managing the data, merging, and
allocation of data in the correct arena. It also safely merges and deep
copies data from the transient arenas into persistent storage that is
never reset until the program completes.
This patch reduces memory by another % over the previous patches,
bringing the total savings over the baseline to 57%. Runtime performance
and benchmarks stay mostly flat with modest improvements.
| Metric | Baseline | Prev | This | Culm% | Seq% |
| :--- | :--- | :--- | :--- | :--- | :--- |
| Time | 920.5s | 991.5s | 987.2s | +7.2% | -0.4% |
| Memory | 86.0G | 40.0G | 36.9G | -57.1% | -8.0% |
| Benchmark | Baseline | Prev | This | Culm% | Seq% |
| :--- | :--- | :--- | :--- | :--- | :--- |
| BM_BitcodeReader_Scale/10 | 67.9us | 72.2us | 72.2us | +6.3% | -0.0% |
| BM_BitcodeReader_Scale/10000 | 70.5ms | 22.5ms | 17.3ms | -75.5% | -23.2% |
| BM_BitcodeReader_Scale/4096 | 23.2ms | 6.6ms | 7.1ms | -69.5% | +7.4% |
| BM_BitcodeReader_Scale/512 | 509.4us | 898.7us | 550.5us | +8.1% | -38.7% |
| BM_BitcodeReader_Scale/64 | 114.8us | 133.7us | 120.8us | +5.2% | -9.6% |
| BM_EmitInfoFunction | 1.6us | 1.9us | 1.8us | +12.9% | -3.3% |
| BM_Index_Insertion/10 | 2.3us | 4.1us | 3.5us | +52.4% | -14.7% |
| BM_Index_Insertion/10000 | 3.1ms | 5.3ms | 4.8ms | +53.5% | -10.0% |
| BM_Index_Insertion/4096 | 1.3ms | 2.1ms | 1.9ms | +50.8% | -9.2% |
| BM_Index_Insertion/512 | 153.6us | 251.8us | 227.0us | +47.8% | -9.9% |
| BM_Index_Insertion/64 | 18.1us | 30.2us | 26.7us | +47.9% | -11.7% |
| BM_JSONGenerator_Scale/10000 | 89.6ms | 81.4ms | 83.4ms | -6.9% | +2.5% |
| BM_JSONGenerator_Scale/4096 | 33.7ms | 31.0ms | 32.4ms | -3.9% | +4.5% |
| BM_Mapper_Scale/10000 | 104.3ms | 112.3ms | 103.5ms | -0.8% | -7.9% |
| BM_Mapper_Scale/4096 | 44.3ms | 45.0ms | 43.8ms | -1.2% | -2.5% |
| BM_Mapper_Scale/512 | 7.6ms | 7.7ms | 7.5ms | -1.3% | -2.4% |
| BM_Mapper_Scale/64 | 3.1ms | 3.0ms | 3.0ms | -1.9% | -0.3% |
| BM_MergeInfos_Scale/10000 | 12.2ms | 575.6us | 500.1us | -95.9% | -13.1% |
| BM_MergeInfos_Scale/2 | 1.9us | 1.8us | 1.8us | -4.4% | -1.7% |
| BM_MergeInfos_Scale/4096 | 2.8ms | 205.3us | 200.4us | -92.8% | -2.4% |
| BM_MergeInfos_Scale/512 | 68.9us | 20.5us | 19.5us | -71.7% | -5.1% |
| BM_MergeInfos_Scale/64 | 10.3us | 3.8us | 4.0us | -60.9% | +4.8% |
| BM_MergeInfos_Scale/8 | 2.8us | 1.9us | 1.9us | -31.7% | -1.8% |
| BM_SerializeFunctionInfo | 25.5us | 25.8us | 26.1us | +2.2% | +1.3% |
---
.../clang-doc/Representation.cpp | 144 ++++++++++++++++--
clang-tools-extra/clang-doc/Representation.h | 7 +
.../clang-doc/tool/ClangDocMain.cpp | 50 +++---
3 files changed, 166 insertions(+), 35 deletions(-)
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 718a466eaeb8a..7fdb0dfe3e5e9 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -115,7 +115,31 @@ static void reduceChildren(llvm::simple_ilist<T> &Children,
auto It = llvm::find_if(
Children, [&](const T &C) { return C.USR == ChildToMerge->USR; });
if (It == Children.end()) {
- T *NewChild = allocatePtr<T>(PersistentArena, std::move(*ChildToMerge));
+ T *NewChild = allocatePtr<T>(PersistentArena, ChildToMerge->USR);
+ NewChild->merge(std::move(*ChildToMerge));
+ Children.push_back(*NewChild);
+ } else {
+ It->merge(std::move(*ChildToMerge));
+ }
+ }
+}
+
+template <>
+void reduceChildren<Reference>(
+ llvm::simple_ilist<Reference> &Children,
+ llvm::simple_ilist<Reference> &&ChildrenToMerge) {
+ while (!ChildrenToMerge.empty()) {
+ Reference *ChildToMerge = &ChildrenToMerge.front();
+ ChildrenToMerge.pop_front();
+
+ auto It = llvm::find_if(Children, [&](const Reference &C) {
+ return C.USR == ChildToMerge->USR;
+ });
+ if (It == Children.end()) {
+ Reference *NewChild = allocatePtr<Reference>(PersistentArena);
+ NewChild->USR = ChildToMerge->USR;
+ NewChild->RefType = ChildToMerge->RefType;
+ NewChild->merge(std::move(*ChildToMerge));
Children.push_back(*NewChild);
} else {
It->merge(std::move(*ChildToMerge));
@@ -130,12 +154,106 @@ static void mergeUnkeyed(Container &Target, Container &&Source) {
auto &Item = Source.front();
Source.pop_front();
if (llvm::none_of(Target, [&](const auto &E) { return E == Item; })) {
- T *NewItem = allocatePtr<T>(PersistentArena, std::move(Item));
+ T *NewItem = allocatePtr<T>(PersistentArena, Item);
+ Target.push_back(*NewItem);
+ }
+ }
+}
+
+template <>
+void mergeUnkeyed<OwningVec<CommentInfo>>(OwningVec<CommentInfo> &Target,
+ OwningVec<CommentInfo> &&Source) {
+ while (!Source.empty()) {
+ auto &Item = Source.front();
+ Source.pop_front();
+ if (llvm::none_of(Target, [&](const auto &E) { return E == Item; })) {
+ CommentInfo *NewItem =
+ allocatePtr<CommentInfo>(PersistentArena, Item, PersistentArena);
Target.push_back(*NewItem);
}
}
}
+llvm::Error mergeSingleInfo(doc::OwnedPtr<doc::Info> &Reduced,
+ doc::OwnedPtr<doc::Info> &&NewInfo,
+ llvm::BumpPtrAllocator &Arena) {
+ if (!Reduced) {
+ switch (NewInfo->IT) {
+ case InfoType::IT_namespace:
+ Reduced = allocatePtr<NamespaceInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_record:
+ Reduced = allocatePtr<RecordInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_enum:
+ Reduced = allocatePtr<EnumInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_function:
+ Reduced = allocatePtr<FunctionInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_typedef:
+ Reduced = allocatePtr<TypedefInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_concept:
+ Reduced = allocatePtr<ConceptInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_variable:
+ Reduced = allocatePtr<VarInfo>(Arena, NewInfo->USR);
+ break;
+ case InfoType::IT_friend:
+ Reduced = allocatePtr<FriendInfo>(Arena, NewInfo->USR);
+ break;
+ default:
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ "unknown info type");
+ }
+ }
+
+ if (Reduced->IT != NewInfo->IT)
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ "info types mismatch");
+
+ switch (Reduced->IT) {
+ case InfoType::IT_namespace:
+ static_cast<NamespaceInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<NamespaceInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_record:
+ static_cast<RecordInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<RecordInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_enum:
+ static_cast<EnumInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<EnumInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_function:
+ static_cast<FunctionInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<FunctionInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_typedef:
+ static_cast<TypedefInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<TypedefInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_concept:
+ static_cast<ConceptInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<ConceptInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_variable:
+ static_cast<VarInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<VarInfo *>(getPtr(NewInfo))));
+ break;
+ case InfoType::IT_friend:
+ static_cast<FriendInfo *>(getPtr(Reduced))
+ ->merge(std::move(*static_cast<FriendInfo *>(getPtr(NewInfo))));
+ break;
+ default:
+ return llvm::createStringError(llvm::inconvertibleErrorCode(),
+ "unknown info type");
+ }
+
+ return llvm::Error::success();
+}
+
// Dispatch function.
llvm::Expected<OwnedPtr<Info>> mergeInfos(OwningPtrArray<Info> &Values) {
if (Values.empty() || !Values[0])
@@ -293,6 +411,8 @@ void Reference::merge(Reference &&Other) {
Name = Other.Name;
if (Path.empty())
Path = Other.Path;
+ if (QualName.empty())
+ QualName = Other.QualName;
if (DocumentationFileName.empty())
DocumentationFileName = Other.DocumentationFileName;
}
@@ -340,8 +460,8 @@ void Info::mergeBase(Info &&Other) {
Name = Other.Name;
if (Path == "")
Path = Other.Path;
- if (Namespace.empty())
- Namespace = std::move(Other.Namespace);
+ if (Namespace.empty() && !Other.Namespace.empty())
+ Namespace = allocateArray(Other.Namespace, PersistentArena);
// Unconditionally extend the description, since each decl may have a comment.
mergeUnkeyed(Description, std::move(Other.Description));
if (ParentUSR == EmptySID)
@@ -374,6 +494,8 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
mergeBase(std::move(Other));
if (MangledName.empty())
MangledName = std::move(Other.MangledName);
+ if (!IsStatic)
+ IsStatic = Other.IsStatic;
}
NamespaceInfo::NamespaceInfo(SymbolID USR, StringRef Name, StringRef Path)
@@ -442,8 +564,8 @@ void RecordInfo::merge(RecordInfo &&Other) {
reduceChildren(Children.Enums, std::move(Other.Children.Enums));
reduceChildren(Children.Typedefs, std::move(Other.Children.Typedefs));
SymbolInfo::merge(std::move(Other));
- if (!Template)
- Template = Other.Template;
+ if (!Template && Other.Template)
+ Template = TemplateInfo(*Other.Template, PersistentArena);
}
EnumValueInfo::EnumValueInfo(const EnumValueInfo &Other,
@@ -461,6 +583,8 @@ void EnumInfo::merge(EnumInfo &&Other) {
assert(mergeable(Other));
if (!Scoped)
Scoped = Other.Scoped;
+ if (!BaseType && Other.BaseType)
+ BaseType = std::move(Other.BaseType);
if (Members.empty() && !Other.Members.empty())
Members = deepCopyArray(Other.Members, PersistentArena);
SymbolInfo::merge(std::move(Other));
@@ -479,8 +603,8 @@ void FunctionInfo::merge(FunctionInfo &&Other) {
if (Params.empty() && !Other.Params.empty())
Params = allocateArray(Other.Params, PersistentArena);
SymbolInfo::merge(std::move(Other));
- if (!Template)
- Template = Other.Template;
+ if (!Template && Other.Template)
+ Template = TemplateInfo(*Other.Template, PersistentArena);
}
void TypedefInfo::merge(TypedefInfo &&Other) {
@@ -489,8 +613,8 @@ void TypedefInfo::merge(TypedefInfo &&Other) {
IsUsing = Other.IsUsing;
if (Underlying.Type.Name == "")
Underlying = Other.Underlying;
- if (!Template)
- Template = Other.Template;
+ if (!Template && Other.Template)
+ Template = TemplateInfo(*Other.Template, PersistentArena);
SymbolInfo::merge(std::move(Other));
}
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 0a6fbd501d5e5..9a323772ef167 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -52,6 +52,7 @@ class ConcurrentStringPool {
ConcurrentStringPool &getGlobalStringPool();
extern thread_local llvm::BumpPtrAllocator TransientArena;
+extern thread_local llvm::BumpPtrAllocator PersistentArena;
inline StringRef internString(const Twine &T) {
if (T.isTriviallyEmpty())
@@ -776,6 +777,12 @@ struct Index : public Reference {
// if they are different.
llvm::Expected<OwnedPtr<Info>> mergeInfos(OwningPtrArray<Info> &Values);
+// Merges a single new Info into an existing Reduced Info (allocating it if
+// needed).
+llvm::Error mergeSingleInfo(doc::OwnedPtr<doc::Info> &Reduced,
+ doc::OwnedPtr<doc::Info> &&NewInfo,
+ llvm::BumpPtrAllocator &Arena);
+
struct ClangDocContext {
ClangDocContext(tooling::ExecutionContext *ECtx, StringRef ProjectName,
bool PublicOnly, StringRef OutDirectory, StringRef SourceRoot,
diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index f627ee5887528..3c38901f4a0f9 100644
--- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -29,6 +29,7 @@
#include "clang/Tooling/CommonOptionsParser.h"
#include "clang/Tooling/Execution.h"
#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -358,15 +359,22 @@ Example usage for a project using a compile commands database:
llvm::hardware_concurrency(ExecutorConcurrency));
{
llvm::TimeTraceScope TS("Reduce");
- for (auto &Group : USRToBitcode) {
- Pool.async([&, &Diags = Diags]() { // time trace decoding bitcode
- if (FTimeTrace)
+ for (const auto &Group : USRToBitcode) {
+ StringRef Key = Group.getKey();
+ std::vector<StringRef> Bitcodes = Group.getValue();
+ Pool.async([Key, Bitcodes, &CDCtx, &Diags, &USRToInfo, &USRToInfoMutex,
+ &IndexMutex, &DiagMutex, &Error, DiagIDBitcodeReading,
+ DiagIDBitcodeMerging]() {
+ if (CDCtx.FTimeTrace)
llvm::timeTraceProfilerInitialize(200, "clang-doc");
- doc::OwningPtrVec<doc::Info> Infos;
+ doc::OwnedPtr<doc::Info> Reduced = nullptr;
{
- llvm::TimeTraceScope Red("decoding bitcode");
- for (auto &Bitcode : Group.getValue()) {
+ llvm::TimeTraceScope Red("decoding and merging bitcode");
+ for (const auto &Bitcode : Bitcodes) {
+
+ llvm::scope_exit ArenaGuard(
+ [] { clang::doc::TransientArena.Reset(); });
llvm::BitstreamCursor Stream(Bitcode);
doc::ClangDocBitcodeReader Reader(Stream, Diags);
auto ReadInfos = Reader.readBitcode();
@@ -378,25 +386,17 @@ Example usage for a project using a compile commands database:
Error = true;
return;
}
- std::move(ReadInfos->begin(), ReadInfos->end(),
- std::back_inserter(Infos));
- }
- } // time trace decoding bitcode
-
- doc::OwnedPtr<doc::Info> Reduced;
-
- {
- llvm::TimeTraceScope Merge("merging bitcode");
- auto ExpReduced = doc::mergeInfos(Infos);
-
- if (!ExpReduced) {
- std::lock_guard<llvm::sys::Mutex> Guard(DiagMutex);
- Diags.Report(DiagIDBitcodeMerging)
- << toString(ExpReduced.takeError());
- return;
+ for (auto &I : *ReadInfos) {
+ if (auto Err = doc::mergeSingleInfo(
+ Reduced, std::move(I), clang::doc::PersistentArena)) {
+ std::lock_guard<llvm::sys::Mutex> Guard(DiagMutex);
+ Diags.Report(DiagIDBitcodeMerging)
+ << toString(std::move(Err));
+ return;
+ }
+ }
}
- Reduced = std::move(*ExpReduced);
- } // time trace merging bitcode
+ } // time trace decoding and merging bitcode
// Add a reference to this Info in the Index
{
@@ -408,7 +408,7 @@ Example usage for a project using a compile commands database:
{
llvm::TimeTraceScope Merge("USRToInfo");
std::lock_guard<llvm::sys::Mutex> Guard(USRToInfoMutex);
- USRToInfo[Group.getKey()] = std::move(Reduced);
+ USRToInfo[Key] = std::move(Reduced);
}
if (CDCtx.FTimeTrace)
More information about the llvm-branch-commits
mailing list