[clang] [clang][Tooling] Reuse standard-library symbol descriptors (PR #202663)
David Zbarsky via cfe-commits
cfe-commits at lists.llvm.org
Thu Jul 2 12:55:23 PDT 2026
https://github.com/dzbarsky updated https://github.com/llvm/llvm-project/pull/202663
>From 7bd79992f8744d6055a3257757f3476416551b55 Mon Sep 17 00:00:00 2001
From: David Zbarsky <dzbarsky at gmail.com>
Date: Tue, 9 Jun 2026 04:54:39 -0400
Subject: [PATCH] [clang][Tooling] Reuse standard-library symbol descriptors
StandardLibrary.cpp expands the generated C and C++ symbol mappings once to initialize SymbolHeaderMapping and a second time to count unique symbols. The generated SymbolMapping arrays also store two pointers and an unsigned integer for every symbol-to-header mapping.
Store each qualified name and header name as one QName\0Header name in EnumStringDef<uint8_t, 1>, with the namespace length in value(). BUILD_ENUM_STRINGS creates pointer-free EnumStringsStorage tables. Split StdSymbolMap.inc when the four-byte EnumString records and string bytes approach the uint16_t offset limit. Count and initialize SymbolHeaderMapping from the same EnumStrings tables.
In an arm64 Release build with assertions, StandardLibrary.cpp.o decreases from 542,568 to 212,344 bytes (-330,224), and its loadable contents decrease from 272,928 to 190,176 bytes (-82,752). __DATA,__const decreases by 184,416 bytes, __TEXT,__cstring decreases by 70,113 bytes, __TEXT,__const increases by 172,302 bytes, and object relocations decrease from 18,041 to 784 (-17,257).
Linked clangd decreases from 122,442,440 to 122,375,944 bytes (-66,496), and stripped clangd decreases from 69,416,496 to 69,350,440 bytes (-66,056). Linked fixups decrease from 176,944 to 159,655 (-17,289).
An exhaustive dump of all public C and C++ symbol-to-header mappings is byte-identical before and after the change (4,463 lines). clangd --check also completes with zero errors.
---
.../Inclusions/Stdlib/StandardLibrary.cpp | 168 ++++++++++++------
.../Inclusions/Stdlib/StdSymbolMap.inc | 4 +
clang/tools/include-mapping/gen_std.py | 56 +++++-
3 files changed, 172 insertions(+), 56 deletions(-)
diff --git a/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp b/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp
index 807a8d8a34ad7..ee44dadefc8dc 100644
--- a/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp
+++ b/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp
@@ -10,11 +10,15 @@
#include "clang/AST/Decl.h"
#include "clang/Basic/LangOptions.h"
#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Enum.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cstdint>
+#include <limits>
#include <optional>
+#include <utility>
namespace clang {
namespace tooling {
@@ -24,6 +28,72 @@ namespace {
// Symbol name -> Symbol::ID, within a namespace.
using NSSymbolMap = llvm::DenseMap<llvm::StringRef, unsigned>;
+using SymbolNamespaceLength = uint8_t;
+using SymbolMappingDef = llvm::EnumStringDef<SymbolNamespaceLength>;
+using SymbolMapping = llvm::EnumString<SymbolNamespaceLength>;
+using SymbolMappingTable = llvm::EnumStrings<SymbolNamespaceLength>;
+static_assert(sizeof(SymbolMapping) == 4,
+ "update the standard symbol table partition size");
+
+template <size_t Size>
+constexpr SymbolNamespaceLength getSymbolNamespaceLength(const char (&)[Size]) {
+ static_assert(Size - 1 <= std::numeric_limits<SymbolNamespaceLength>::max(),
+ "symbol namespace length does not fit in one byte");
+ return static_cast<SymbolNamespaceLength>(Size - 1);
+}
+
+static constexpr SymbolMappingDef CSymbolDefs[] = {
+#define SYMBOL(Name, NS, Header) \
+ {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \
+ getSymbolNamespaceLength(#NS)},
+#include "CSpecialSymbolMap.inc"
+#include "CSymbolMap.inc"
+#undef SYMBOL
+};
+static constexpr auto CSymbols = BUILD_ENUM_STRINGS(CSymbolDefs);
+
+static constexpr SymbolMappingDef CXXSpecialSymbolDefs[] = {
+#define SYMBOL(Name, NS, Header) \
+ {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \
+ getSymbolNamespaceLength(#NS)},
+#include "StdSpecialSymbolMap.inc"
+#undef SYMBOL
+};
+static constexpr auto CXXSpecialSymbols =
+ BUILD_ENUM_STRINGS(CXXSpecialSymbolDefs);
+
+#define SYMBOL_MAP_BEGIN(Index) \
+ static constexpr SymbolMappingDef CXXSymbolDefs##Index[] = {
+#define SYMBOL(Name, NS, Header) \
+ {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \
+ getSymbolNamespaceLength(#NS)},
+#define SYMBOL_MAP_END(Index) \
+ } \
+ ; \
+ static constexpr auto CXXSymbols##Index = \
+ BUILD_ENUM_STRINGS(CXXSymbolDefs##Index);
+#include "StdSymbolMap.inc"
+#undef SYMBOL_MAP_END
+#undef SYMBOL
+#undef SYMBOL_MAP_BEGIN
+
+static constexpr SymbolMappingDef CXXTsSymbolDefs[] = {
+#define SYMBOL(Name, NS, Header) \
+ {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \
+ getSymbolNamespaceLength(#NS)},
+#include "StdTsSymbolMap.inc"
+#undef SYMBOL
+};
+static constexpr auto CXXTsSymbols = BUILD_ENUM_STRINGS(CXXTsSymbolDefs);
+
+static std::pair<StringRef, StringRef>
+splitSymbolMapping(const SymbolMapping &Mapping) {
+ StringRef Name = Mapping.name();
+ size_t Delimiter = Name.find('\0');
+ assert(Delimiter != StringRef::npos && "missing symbol mapping delimiter");
+ return {Name.take_front(Delimiter), Name.drop_front(Delimiter + 1)};
+}
+
// A Mapping per language.
struct SymbolHeaderMapping {
llvm::StringRef *HeaderNames = nullptr;
@@ -54,37 +124,27 @@ static const SymbolHeaderMapping *getMappingPerLang(Lang L) {
return LanguageMappings[static_cast<unsigned>(L)];
}
-static int countSymbols(Lang Language) {
- ArrayRef<const char *> Symbols;
-#define SYMBOL(Name, NS, Header) #NS #Name,
- switch (Language) {
- case Lang::C: {
- static constexpr const char *CSymbols[] = {
-#include "CSpecialSymbolMap.inc"
-#include "CSymbolMap.inc"
- };
- Symbols = CSymbols;
- break;
- }
- case Lang::CXX: {
- static constexpr const char *CXXSymbols[] = {
-#include "StdSpecialSymbolMap.inc"
-#include "StdSymbolMap.inc"
-#include "StdTsSymbolMap.inc"
- };
- Symbols = CXXSymbols;
- break;
- }
+static unsigned countSymbols(ArrayRef<SymbolMappingTable> SymbolMappingTables) {
+ unsigned Count = 0;
+ StringRef Previous;
+ for (const SymbolMappingTable &Table : SymbolMappingTables) {
+ for (const SymbolMapping &Mapping : Table) {
+ StringRef QName = splitSymbolMapping(Mapping).first;
+ if (Previous != QName) {
+ ++Count;
+ Previous = QName;
+ }
+ }
}
-#undef SYMBOL
- return llvm::DenseSet<StringRef>(llvm::from_range, Symbols).size();
+ return Count;
}
-static int initialize(Lang Language) {
+static int initialize(Lang Language,
+ ArrayRef<SymbolMappingTable> SymbolMappingTables) {
SymbolHeaderMapping *Mapping = new SymbolHeaderMapping();
LanguageMappings[static_cast<unsigned>(Language)] = Mapping;
- unsigned SymCount = countSymbols(Language);
+ unsigned SymCount = countSymbols(SymbolMappingTables);
Mapping->SymbolCount = SymCount;
Mapping->SymbolNames =
new std::remove_reference_t<decltype(*Mapping->SymbolNames)>[SymCount];
@@ -137,36 +197,12 @@ static int initialize(Lang Language) {
NSSymbols.try_emplace(QName.drop_front(NSLen), SymIndex);
};
- struct Symbol {
- const char *QName;
- unsigned NSLen;
- const char *HeaderName;
- };
-#define SYMBOL(Name, NS, Header) \
- {#NS #Name, static_cast<decltype(Symbol::NSLen)>(StringRef(#NS).size()), \
- #Header},
- switch (Language) {
- case Lang::C: {
- static constexpr Symbol CSymbols[] = {
-#include "CSpecialSymbolMap.inc"
-#include "CSymbolMap.inc"
- };
- for (const Symbol &S : CSymbols)
- Add(S.QName, S.NSLen, S.HeaderName);
- break;
- }
- case Lang::CXX: {
- static constexpr Symbol CXXSymbols[] = {
-#include "StdSpecialSymbolMap.inc"
-#include "StdSymbolMap.inc"
-#include "StdTsSymbolMap.inc"
- };
- for (const Symbol &S : CXXSymbols)
- Add(S.QName, S.NSLen, S.HeaderName);
- break;
- }
+ for (const SymbolMappingTable &Table : SymbolMappingTables) {
+ for (const SymbolMapping &Mapping : Table) {
+ auto [QName, HeaderName] = splitSymbolMapping(Mapping);
+ Add(QName, Mapping.value(), HeaderName);
+ }
}
-#undef SYMBOL
Mapping->HeaderNames = new llvm::StringRef[Mapping->HeaderIDs->size()];
for (const auto &E : *Mapping->HeaderIDs)
@@ -175,6 +211,28 @@ static int initialize(Lang Language) {
return 0;
}
+static int initialize(Lang Language) {
+ switch (Language) {
+ case Lang::C: {
+ const SymbolMappingTable SymbolMappingTables[] = {CSymbols};
+ return initialize(Language, SymbolMappingTables);
+ }
+ case Lang::CXX: {
+ const SymbolMappingTable SymbolMappingTables[] = {CXXSpecialSymbols,
+#define SYMBOL(Name, NS, Header)
+#define SYMBOL_MAP_BEGIN(Index) CXXSymbols##Index,
+#define SYMBOL_MAP_END(Index)
+#include "StdSymbolMap.inc"
+#undef SYMBOL_MAP_END
+#undef SYMBOL_MAP_BEGIN
+#undef SYMBOL
+ CXXTsSymbols};
+ return initialize(Language, SymbolMappingTables);
+ }
+ }
+ llvm_unreachable("unknown language");
+}
+
static void ensureInitialized() {
static int Dummy = []() {
for (unsigned L = 0; L <= static_cast<unsigned>(Lang::LastValue); ++L)
diff --git a/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc b/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc
index c1927180d3397..45329d83458d4 100644
--- a/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc
+++ b/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc
@@ -9,6 +9,7 @@
// Generated from cppreference offline HTML book (modified on 2024-11-10).
//===----------------------------------------------------------------------===//
+SYMBOL_MAP_BEGIN(0)
SYMBOL(ATOMIC_BOOL_LOCK_FREE, None, <atomic>)
SYMBOL(ATOMIC_CHAR16_T_LOCK_FREE, None, <atomic>)
SYMBOL(ATOMIC_CHAR32_T_LOCK_FREE, None, <atomic>)
@@ -2289,6 +2290,8 @@ SYMBOL(nextafterf, std::, <cmath>)
SYMBOL(nextafterf, None, <cmath>)
SYMBOL(nextafterf, None, <math.h>)
SYMBOL(nextafterl, std::, <cmath>)
+SYMBOL_MAP_END(0)
+SYMBOL_MAP_BEGIN(1)
SYMBOL(nextafterl, None, <cmath>)
SYMBOL(nextafterl, None, <math.h>)
SYMBOL(nexttoward, std::, <cmath>)
@@ -3985,3 +3988,4 @@ SYMBOL(transform, std::views::, <ranges>)
SYMBOL(values, std::views::, <ranges>)
SYMBOL(zip, std::views::, <ranges>)
SYMBOL(zip_transform, std::views::, <ranges>)
+SYMBOL_MAP_END(1)
diff --git a/clang/tools/include-mapping/gen_std.py b/clang/tools/include-mapping/gen_std.py
index f362227bc6aab..aac87a436b372 100755
--- a/clang/tools/include-mapping/gen_std.py
+++ b/clang/tools/include-mapping/gen_std.py
@@ -55,6 +55,58 @@
//===----------------------------------------------------------------------===//
"""
+# EnumString<uint8_t, 1> stores a 4-byte record for each string. Its 16-bit
+# offset must address the string data from the record.
+MAX_SYMBOL_TABLE_STORAGE_BYTES = (1 << 16) - 1
+SYMBOL_TABLE_RECORD_BYTES = 4
+
+
+def SymbolTableRowSize(symbol_row):
+ name, namespace, header = symbol_row
+ string_size = len(("%s%s\0%s" % (namespace, name, header)).encode("utf-8"))
+ return SYMBOL_TABLE_RECORD_BYTES + string_size
+
+
+def EmitSymbolRows(symbol_rows, partition_symbol_tables):
+ if not partition_symbol_tables:
+ for name, namespace, header in symbol_rows:
+ print("SYMBOL(%s, %s, %s)" % (name, namespace, header))
+ return
+
+ symbol_groups = []
+ for symbol_row in symbol_rows:
+ qualified_name = symbol_row[1], symbol_row[0]
+ if not symbol_groups or symbol_groups[-1][0] != qualified_name:
+ symbol_groups.append((qualified_name, []))
+ symbol_groups[-1][1].append(symbol_row)
+
+ symbol_tables = []
+ current_symbol_table = []
+ current_symbol_table_size = 0
+ for _, symbol_group in symbol_groups:
+ group_size = sum(SymbolTableRowSize(row) for row in symbol_group)
+ if group_size > MAX_SYMBOL_TABLE_STORAGE_BYTES:
+ raise ValueError("symbol mapping group exceeds symbol table limit")
+ if (
+ current_symbol_table
+ and current_symbol_table_size + group_size > MAX_SYMBOL_TABLE_STORAGE_BYTES
+ ):
+ symbol_tables.append(current_symbol_table)
+ current_symbol_table = []
+ current_symbol_table_size = 0
+ current_symbol_table.extend(symbol_group)
+ current_symbol_table_size += group_size
+ if current_symbol_table:
+ symbol_tables.append(current_symbol_table)
+ if not symbol_tables:
+ symbol_tables.append([])
+
+ for table_index, symbol_table in enumerate(symbol_tables):
+ print("SYMBOL_MAP_BEGIN(%d)" % table_index)
+ for name, namespace, header in symbol_table:
+ print("SYMBOL(%s, %s, %s)" % (name, namespace, header))
+ print("SYMBOL_MAP_END(%d)" % table_index)
+
def ParseArg():
parser = argparse.ArgumentParser(description="Generate StdGen file")
@@ -263,6 +315,7 @@ def main():
exit("Path %s doesn't exist!" % symbol_index_root)
symbols = cppreference_parser.GetSymbols(parse_pages)
+ symbol_rows = []
# We don't have version information from the unzipped offline HTML files.
# so we use the modified time of the symbol_index.html as the version.
@@ -279,7 +332,7 @@ def main():
s.headers.extend(AdditionalHeadersForIOSymbols(s))
for header in s.headers:
# SYMBOL(unqualified_name, namespace, header)
- print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace, header))
+ symbol_rows.append((s.name, s.namespace, header))
elif len(symbol.headers) == 0:
sys.stderr.write("No header found for symbol %s\n" % symbol.name)
else:
@@ -288,6 +341,7 @@ def main():
"Ambiguous header for symbol %s: %s\n"
% (symbol.name, ", ".join(symbol.headers))
)
+ EmitSymbolRows(symbol_rows, args.symbols == "cpp")
if __name__ == "__main__":
More information about the cfe-commits
mailing list