[clang-tools-extra] [clangd] Speed up loading the background index from disk (PR #237760)
Nikita Taranov via cfe-commits
cfe-commits at lists.llvm.org
Sun Nov 1 11:51:15 PST 2026
https://github.com/nickitat updated https://github.com/llvm/llvm-project/pull/237760
>From 07d52699e1b110c4eb1a5cdd2ca059dcc1ea823f Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov at clickhouse.com>
Date: Sun, 1 Nov 2026 18:34:41 +0000
Subject: [PATCH 1/5] [clangd] Speed up loading the background index from disk
Two hot spots showed up when profiling how clangd loads a large background
index (17k TUs, 41k shards, 447 MB) at startup:
- `Dex` re-inserted every entry of the refs map built by
`FileSymbols::buildIndex` into its own `DenseMap` of the same type. It now
takes the map over when it is passed by rvalue. This was 18.6% of the load.
- Refs read from an index file went through `RefSlab::Builder`, which interns
the file URI and inserts into a `DenseSet` for every ref, then sorts
everything again. Refs on disk are already grouped by symbol, so add
`RefSlab::BundleBuilder`, which takes whole per-symbol bundles, interns each
URI once per distinct string, and keeps the existing order. If a symbol
appears in several bundles it falls back to `RefSlab::Builder`, so the
result is the same. This was about 12% of the load.
Time from enqueueing the project to serving the loaded index, on a warm page
cache: 5.4-5.6 s before, 4.0-4.1 s after (-26%). Results of
`workspace/symbol` and `textDocument/references` queries are identical.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
clang-tools-extra/clangd/index/Ref.cpp | 39 +++++++++++++++
clang-tools-extra/clangd/index/Ref.h | 21 ++++++++
.../clangd/index/Serialization.cpp | 5 +-
clang-tools-extra/clangd/index/dex/Dex.h | 11 ++++-
.../clangd/unittests/IndexTests.cpp | 49 +++++++++++++++++++
5 files changed, 120 insertions(+), 5 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Ref.cpp b/clang-tools-extra/clangd/index/Ref.cpp
index edbfb95b528400..41bb3de7fb78a1 100644
--- a/clang-tools-extra/clangd/index/Ref.cpp
+++ b/clang-tools-extra/clangd/index/Ref.cpp
@@ -39,6 +39,45 @@ void RefSlab::Builder::insert(const SymbolID &ID, const Ref &S) {
Entries.insert(std::move(E));
}
+void RefSlab::BundleBuilder::insert(const SymbolID &ID,
+ llvm::ArrayRef<Ref> SymbolRefs) {
+ if (SymbolRefs.empty())
+ return;
+ Ref *Copy = Arena.Allocate<Ref>(SymbolRefs.size());
+ for (size_t I = 0; I < SymbolRefs.size(); ++I) {
+ Ref R = SymbolRefs[I];
+ const char *&Interned = InternedURIs[R.Location.FileURI];
+ if (!Interned)
+ Interned = UniqueStrings.save(R.Location.FileURI).data();
+ R.Location.FileURI = Interned;
+ new (&Copy[I]) Ref(R);
+ }
+ llvm::MutableArrayRef<Ref> Stored(Copy, SymbolRefs.size());
+ if (!llvm::is_sorted(Stored))
+ llvm::sort(Stored); // By file, affects xrefs display order.
+ Refs.emplace_back(ID, Stored);
+ NumRefs += SymbolRefs.size();
+}
+
+RefSlab RefSlab::BundleBuilder::build() && {
+ // Keep the invariant of Builder: sorted by SymbolID, one entry per symbol.
+ if (!llvm::is_sorted(Refs, llvm::less_first()))
+ llvm::stable_sort(Refs, llvm::less_first());
+ std::vector<value_type> Merged;
+ bool HasDuplicates = false;
+ for (size_t I = 1; I < Refs.size() && !HasDuplicates; ++I)
+ HasDuplicates = Refs[I - 1].first == Refs[I].first;
+ if (!HasDuplicates)
+ return RefSlab(std::move(Refs), std::move(Arena), NumRefs);
+ // Rare: the same symbol appeared in several bundles. Fall back to Builder,
+ // which also removes duplicate refs.
+ Builder B;
+ for (const auto &[ID, SymbolRefs] : Refs)
+ for (const Ref &R : SymbolRefs)
+ B.insert(ID, R);
+ return std::move(B).build();
+}
+
RefSlab RefSlab::Builder::build() && {
std::vector<std::pair<SymbolID, llvm::ArrayRef<Ref>>> Result;
// We'll reuse the arena, as it only has unique strings and we need them all.
diff --git a/clang-tools-extra/clangd/index/Ref.h b/clang-tools-extra/clangd/index/Ref.h
index b71f09611fed1f..d71d9c7ef916c0 100644
--- a/clang-tools-extra/clangd/index/Ref.h
+++ b/clang-tools-extra/clangd/index/Ref.h
@@ -11,6 +11,7 @@
#include "index/SymbolID.h"
#include "index/SymbolLocation.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/StringSaver.h"
@@ -131,6 +132,26 @@ class RefSlab {
sizeof(value_type) * Refs.capacity();
}
+ /// Builds a RefSlab from refs that are already grouped by symbol, such as
+ /// the ones read from an index file. Unlike Builder, does not deduplicate
+ /// individual refs, which makes it much cheaper.
+ class BundleBuilder {
+ public:
+ BundleBuilder() : UniqueStrings(Arena) {}
+ /// Adds all refs of \p ID. Deep copy: strings will be owned by the slab.
+ void insert(const SymbolID &ID, llvm::ArrayRef<Ref> SymbolRefs);
+ RefSlab build() &&;
+
+ private:
+ llvm::BumpPtrAllocator Arena;
+ llvm::UniqueStringSaver UniqueStrings; // Contents on the arena.
+ /// Input FileURI pointer -> interned copy. The input usually points into
+ /// a string table, so most lookups hit without hashing the string.
+ llvm::DenseMap<const char *, const char *> InternedURIs;
+ std::vector<value_type> Refs;
+ size_t NumRefs = 0;
+ };
+
/// RefSlab::Builder is a mutable container that can 'freeze' to RefSlab.
class Builder {
public:
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 764ec7903aecb9..508c9783e74e0f 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -518,11 +518,10 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
}
if (Chunks.count("refs")) {
Reader RefsReader(Chunks.lookup("refs"));
- RefSlab::Builder Refs;
+ RefSlab::BundleBuilder Refs;
while (!RefsReader.eof()) {
auto RefsBundle = readRefs(RefsReader, Strings->Strings);
- for (const auto &Ref : RefsBundle.second) // FIXME: bulk insert?
- Refs.insert(RefsBundle.first, Ref);
+ Refs.insert(RefsBundle.first, RefsBundle.second);
}
if (RefsReader.err())
return error("malformed or truncated refs");
diff --git a/clang-tools-extra/clangd/index/dex/Dex.h b/clang-tools-extra/clangd/index/dex/Dex.h
index 1ea7d8c06c67c5..70551cbb140c04 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.h
+++ b/clang-tools-extra/clangd/index/dex/Dex.h
@@ -41,8 +41,15 @@ class Dex : public SymbolIndex {
: Corpus(0) {
for (auto &&Sym : Symbols)
this->Symbols.push_back(&Sym);
- for (auto &&Ref : Refs)
- this->Refs.try_emplace(Ref.first, Ref.second);
+ if constexpr (std::is_same_v<std::decay_t<RefsRange>,
+ decltype(this->Refs)> &&
+ !std::is_lvalue_reference_v<RefsRange>) {
+ // Take over a map of the right type instead of rehashing every entry.
+ this->Refs = std::move(Refs);
+ } else {
+ for (auto &&Ref : Refs)
+ this->Refs.try_emplace(Ref.first, Ref.second);
+ }
for (auto &&Rel : Relations) {
this->Relations[std::make_pair(Rel.Subject,
static_cast<uint8_t>(Rel.Predicate))]
diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp
index a66680d39c87d2..3e790c0b4d8f19 100644
--- a/clang-tools-extra/clangd/unittests/IndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp
@@ -514,6 +514,55 @@ TEST(MergeIndexTest, Refs) {
fileURI("unittest:///test2.cc"))))));
}
+TEST(RefSlabTest, BundleBuilderMatchesBuilder) {
+ SymbolID A("a"), B("b"), C("c");
+ std::string FileX = "file:///x.cc", FileY = "file:///y.cc";
+ auto MakeRef = [](const std::string &File, uint32_t Line) {
+ Ref R;
+ R.Location.FileURI = File.c_str();
+ R.Location.Start.setLine(Line);
+ R.Location.End.setLine(Line);
+ R.Kind = RefKind::Reference;
+ return R;
+ };
+ // Bundles out of SymbolID order, refs out of order within a bundle, and
+ // symbol C split over two bundles that share a ref.
+ std::vector<std::pair<SymbolID, std::vector<Ref>>> Bundles = {
+ {B, {MakeRef(FileY, 2), MakeRef(FileX, 1)}},
+ {C, {MakeRef(FileX, 5)}},
+ {A, {MakeRef(FileX, 3)}},
+ {C, {MakeRef(FileY, 4), MakeRef(FileX, 5)}},
+ };
+ auto Build = [&](bool WithDuplicateSymbol) {
+ RefSlab::Builder Expected;
+ RefSlab::BundleBuilder Actual;
+ for (const auto &[ID, Refs] : Bundles) {
+ if (!WithDuplicateSymbol && &Bundles.back().second == &Refs)
+ continue;
+ for (const Ref &R : Refs)
+ Expected.insert(ID, R);
+ Actual.insert(ID, Refs);
+ }
+ return std::make_pair(std::move(Expected).build(),
+ std::move(Actual).build());
+ };
+ for (bool WithDuplicateSymbol : {false, true}) {
+ auto [Expected, Actual] = Build(WithDuplicateSymbol);
+ // Strings must be owned by the slabs.
+ std::fill(FileX.begin(), FileX.end(), '?');
+ std::fill(FileY.begin(), FileY.end(), '?');
+ EXPECT_EQ(Expected.numRefs(), Actual.numRefs());
+ ASSERT_EQ(Expected.size(), Actual.size());
+ for (size_t I = 0; I < Expected.size(); ++I) {
+ auto ExpectedIt = Expected.begin() + I, ActualIt = Actual.begin() + I;
+ EXPECT_EQ(ExpectedIt->first, ActualIt->first);
+ EXPECT_EQ(ExpectedIt->second, ActualIt->second);
+ }
+ FileX = "file:///x.cc";
+ FileY = "file:///y.cc";
+ }
+}
+
TEST(MergeIndexTest, IndexedFiles) {
SymbolSlab DynSymbols;
RefSlab DynRefs;
>From 6212c7944b46adaedd8f4b752f3af2c236983ec6 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov at clickhouse.com>
Date: Sun, 1 Nov 2026 19:10:57 +0000
Subject: [PATCH 2/5] [clangd] Intern ref file URIs by content in
RefSlab::BundleBuilder
`BundleBuilder` cached interned file URIs by the caller's pointer for its
whole lifetime. A caller that reuses a buffer for a different URI got the
first URI back, which breaks the deep-copy contract of `insert`. Intern by
content, like `RefSlab::Builder` does. This costs about 0.1 s of the 4.4 s
it takes to load a 950 MB index.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
clang-tools-extra/clangd/index/Ref.cpp | 5 +----
clang-tools-extra/clangd/index/Ref.h | 4 ----
.../clangd/unittests/IndexTests.cpp | 20 +++++++++++++++++++
3 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Ref.cpp b/clang-tools-extra/clangd/index/Ref.cpp
index 41bb3de7fb78a1..9a2de4c834ebe5 100644
--- a/clang-tools-extra/clangd/index/Ref.cpp
+++ b/clang-tools-extra/clangd/index/Ref.cpp
@@ -46,10 +46,7 @@ void RefSlab::BundleBuilder::insert(const SymbolID &ID,
Ref *Copy = Arena.Allocate<Ref>(SymbolRefs.size());
for (size_t I = 0; I < SymbolRefs.size(); ++I) {
Ref R = SymbolRefs[I];
- const char *&Interned = InternedURIs[R.Location.FileURI];
- if (!Interned)
- Interned = UniqueStrings.save(R.Location.FileURI).data();
- R.Location.FileURI = Interned;
+ R.Location.FileURI = UniqueStrings.save(R.Location.FileURI).data();
new (&Copy[I]) Ref(R);
}
llvm::MutableArrayRef<Ref> Stored(Copy, SymbolRefs.size());
diff --git a/clang-tools-extra/clangd/index/Ref.h b/clang-tools-extra/clangd/index/Ref.h
index d71d9c7ef916c0..fcd0b6849dfc1d 100644
--- a/clang-tools-extra/clangd/index/Ref.h
+++ b/clang-tools-extra/clangd/index/Ref.h
@@ -11,7 +11,6 @@
#include "index/SymbolID.h"
#include "index/SymbolLocation.h"
-#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/StringSaver.h"
@@ -145,9 +144,6 @@ class RefSlab {
private:
llvm::BumpPtrAllocator Arena;
llvm::UniqueStringSaver UniqueStrings; // Contents on the arena.
- /// Input FileURI pointer -> interned copy. The input usually points into
- /// a string table, so most lookups hit without hashing the string.
- llvm::DenseMap<const char *, const char *> InternedURIs;
std::vector<value_type> Refs;
size_t NumRefs = 0;
};
diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp
index 3e790c0b4d8f19..7f8df6d5cdabf5 100644
--- a/clang-tools-extra/clangd/unittests/IndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp
@@ -563,6 +563,26 @@ TEST(RefSlabTest, BundleBuilderMatchesBuilder) {
}
}
+TEST(RefSlabTest, BundleBuilderCopiesReusedBuffers) {
+ // The same buffer holds a different file URI for each insert.
+ char Buffer[] = "file:///x.cc";
+ Ref R;
+ R.Location.FileURI = Buffer;
+ RefSlab::BundleBuilder Builder;
+ Builder.insert(SymbolID("a"), {R});
+ Buffer[8] = 'y';
+ Builder.insert(SymbolID("b"), {R});
+ Buffer[8] = '?';
+ RefSlab Slab = std::move(Builder).build();
+ llvm::StringMap<SymbolID> FileToSymbol;
+ for (const auto &[ID, Refs] : Slab)
+ for (const Ref &Ref : Refs)
+ FileToSymbol.try_emplace(Ref.Location.FileURI, ID);
+ EXPECT_EQ(FileToSymbol.size(), 2u);
+ EXPECT_EQ(FileToSymbol.lookup("file:///x.cc"), SymbolID("a"));
+ EXPECT_EQ(FileToSymbol.lookup("file:///y.cc"), SymbolID("b"));
+}
+
TEST(MergeIndexTest, IndexedFiles) {
SymbolSlab DynSymbols;
RefSlab DynRefs;
>From 18f244b70b60b266377af482d10c9baac8a5556e Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov at clickhouse.com>
Date: Sun, 1 Nov 2026 19:26:29 +0000
Subject: [PATCH 3/5] [clangd] Forward the refs map into Dex instead of
special-casing rvalues
`std::forward` covers both cases: an rvalue map is moved, and an lvalue map of
the same type is copied, which is still cheaper than re-inserting every entry.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
clang-tools-extra/clangd/index/dex/Dex.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/clang-tools-extra/clangd/index/dex/Dex.h b/clang-tools-extra/clangd/index/dex/Dex.h
index 70551cbb140c04..1b9cbbb62b2050 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.h
+++ b/clang-tools-extra/clangd/index/dex/Dex.h
@@ -42,10 +42,10 @@ class Dex : public SymbolIndex {
for (auto &&Sym : Symbols)
this->Symbols.push_back(&Sym);
if constexpr (std::is_same_v<std::decay_t<RefsRange>,
- decltype(this->Refs)> &&
- !std::is_lvalue_reference_v<RefsRange>) {
- // Take over a map of the right type instead of rehashing every entry.
- this->Refs = std::move(Refs);
+ decltype(this->Refs)>) {
+ // A map of the right type can be taken as is instead of rehashing every
+ // entry.
+ this->Refs = std::forward<RefsRange>(Refs);
} else {
for (auto &&Ref : Refs)
this->Refs.try_emplace(Ref.first, Ref.second);
>From a1ed6e4e85c74ca045514be643390a90a65d29f2 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov at clickhouse.com>
Date: Sun, 1 Nov 2026 19:29:49 +0000
Subject: [PATCH 4/5] [clangd] Take the Dex refs fast path whenever the map is
assignable
Check that the refs argument can be assigned to the map instead of requiring
the exact map type. `Dex::build` still passes a `RefSlab`, which is not
assignable and keeps using the loop.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
clang-tools-extra/clangd/index/dex/Dex.h | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/clang-tools-extra/clangd/index/dex/Dex.h b/clang-tools-extra/clangd/index/dex/Dex.h
index 1b9cbbb62b2050..c106a16aba5c84 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.h
+++ b/clang-tools-extra/clangd/index/dex/Dex.h
@@ -41,10 +41,8 @@ class Dex : public SymbolIndex {
: Corpus(0) {
for (auto &&Sym : Symbols)
this->Symbols.push_back(&Sym);
- if constexpr (std::is_same_v<std::decay_t<RefsRange>,
- decltype(this->Refs)>) {
- // A map of the right type can be taken as is instead of rehashing every
- // entry.
+ if constexpr (std::is_assignable_v<decltype(this->Refs) &, RefsRange>) {
+ // Take a map that can be assigned as is instead of rehashing every entry.
this->Refs = std::forward<RefsRange>(Refs);
} else {
for (auto &&Ref : Refs)
>From 36239554a921bf5055e00f17d29c54763b62bbb4 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov at clickhouse.com>
Date: Sun, 1 Nov 2026 19:51:01 +0000
Subject: [PATCH 5/5] [clangd] Require refs in RefSlab order instead of
repairing them
Refs in an index file are always written from a `RefSlab`: sorted by symbol,
one bundle per symbol, each bundle sorted. `RefSlab::BundleBuilder` used to
re-check and repair the order, and fall back to `RefSlab::Builder` for a
repeated symbol, which never happens for files clangd writes. Make that order
a precondition of `BundleBuilder`, and reject index files that violate it in
`readRIFF` as malformed, like other inconsistencies there.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
clang-tools-extra/clangd/index/Ref.cpp | 25 ++-----
clang-tools-extra/clangd/index/Ref.h | 8 ++-
.../clangd/index/Serialization.cpp | 8 +++
.../clangd/unittests/IndexTests.cpp | 72 +++++++++----------
.../clangd/unittests/SerializationTests.cpp | 25 +++++++
5 files changed, 77 insertions(+), 61 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Ref.cpp b/clang-tools-extra/clangd/index/Ref.cpp
index 9a2de4c834ebe5..5c431060494024 100644
--- a/clang-tools-extra/clangd/index/Ref.cpp
+++ b/clang-tools-extra/clangd/index/Ref.cpp
@@ -41,6 +41,9 @@ void RefSlab::Builder::insert(const SymbolID &ID, const Ref &S) {
void RefSlab::BundleBuilder::insert(const SymbolID &ID,
llvm::ArrayRef<Ref> SymbolRefs) {
+ assert((Refs.empty() || Refs.back().first < ID) &&
+ "Symbols must be added in increasing order");
+ assert(llvm::is_sorted(SymbolRefs) && "Refs of a symbol must be sorted");
if (SymbolRefs.empty())
return;
Ref *Copy = Arena.Allocate<Ref>(SymbolRefs.size());
@@ -49,30 +52,12 @@ void RefSlab::BundleBuilder::insert(const SymbolID &ID,
R.Location.FileURI = UniqueStrings.save(R.Location.FileURI).data();
new (&Copy[I]) Ref(R);
}
- llvm::MutableArrayRef<Ref> Stored(Copy, SymbolRefs.size());
- if (!llvm::is_sorted(Stored))
- llvm::sort(Stored); // By file, affects xrefs display order.
- Refs.emplace_back(ID, Stored);
+ Refs.emplace_back(ID, llvm::ArrayRef<Ref>(Copy, SymbolRefs.size()));
NumRefs += SymbolRefs.size();
}
RefSlab RefSlab::BundleBuilder::build() && {
- // Keep the invariant of Builder: sorted by SymbolID, one entry per symbol.
- if (!llvm::is_sorted(Refs, llvm::less_first()))
- llvm::stable_sort(Refs, llvm::less_first());
- std::vector<value_type> Merged;
- bool HasDuplicates = false;
- for (size_t I = 1; I < Refs.size() && !HasDuplicates; ++I)
- HasDuplicates = Refs[I - 1].first == Refs[I].first;
- if (!HasDuplicates)
- return RefSlab(std::move(Refs), std::move(Arena), NumRefs);
- // Rare: the same symbol appeared in several bundles. Fall back to Builder,
- // which also removes duplicate refs.
- Builder B;
- for (const auto &[ID, SymbolRefs] : Refs)
- for (const Ref &R : SymbolRefs)
- B.insert(ID, R);
- return std::move(B).build();
+ return RefSlab(std::move(Refs), std::move(Arena), NumRefs);
}
RefSlab RefSlab::Builder::build() && {
diff --git a/clang-tools-extra/clangd/index/Ref.h b/clang-tools-extra/clangd/index/Ref.h
index fcd0b6849dfc1d..9d621de1c5cd7c 100644
--- a/clang-tools-extra/clangd/index/Ref.h
+++ b/clang-tools-extra/clangd/index/Ref.h
@@ -131,13 +131,15 @@ class RefSlab {
sizeof(value_type) * Refs.capacity();
}
- /// Builds a RefSlab from refs that are already grouped by symbol, such as
- /// the ones read from an index file. Unlike Builder, does not deduplicate
- /// individual refs, which makes it much cheaper.
+ /// Builds a RefSlab from refs that are already in RefSlab order, such as the
+ /// ones read from an index file. Much cheaper than Builder, which has to
+ /// group, deduplicate and sort refs one by one.
class BundleBuilder {
public:
BundleBuilder() : UniqueStrings(Arena) {}
/// Adds all refs of \p ID. Deep copy: strings will be owned by the slab.
+ /// Symbols must be added in strictly increasing order of their IDs, and
+ /// \p SymbolRefs must be sorted.
void insert(const SymbolID &ID, llvm::ArrayRef<Ref> SymbolRefs);
RefSlab build() &&;
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 508c9783e74e0f..17f6354653f012 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -23,6 +23,7 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdint>
+#include <optional>
#include <vector>
namespace clang {
@@ -519,8 +520,15 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
if (Chunks.count("refs")) {
Reader RefsReader(Chunks.lookup("refs"));
RefSlab::BundleBuilder Refs;
+ std::optional<SymbolID> PrevID;
while (!RefsReader.eof()) {
auto RefsBundle = readRefs(RefsReader, Strings->Strings);
+ // Refs are written from a RefSlab: sorted by symbol, one bundle per
+ // symbol, and each bundle sorted.
+ if ((PrevID && !(*PrevID < RefsBundle.first)) ||
+ !llvm::is_sorted(RefsBundle.second))
+ return error("malformed refs: not sorted or repeated symbol");
+ PrevID = RefsBundle.first;
Refs.insert(RefsBundle.first, RefsBundle.second);
}
if (RefsReader.err())
diff --git a/clang-tools-extra/clangd/unittests/IndexTests.cpp b/clang-tools-extra/clangd/unittests/IndexTests.cpp
index 7f8df6d5cdabf5..4e66743b3cb03c 100644
--- a/clang-tools-extra/clangd/unittests/IndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IndexTests.cpp
@@ -515,7 +515,6 @@ TEST(MergeIndexTest, Refs) {
}
TEST(RefSlabTest, BundleBuilderMatchesBuilder) {
- SymbolID A("a"), B("b"), C("c");
std::string FileX = "file:///x.cc", FileY = "file:///y.cc";
auto MakeRef = [](const std::string &File, uint32_t Line) {
Ref R;
@@ -525,53 +524,50 @@ TEST(RefSlabTest, BundleBuilderMatchesBuilder) {
R.Kind = RefKind::Reference;
return R;
};
- // Bundles out of SymbolID order, refs out of order within a bundle, and
- // symbol C split over two bundles that share a ref.
std::vector<std::pair<SymbolID, std::vector<Ref>>> Bundles = {
- {B, {MakeRef(FileY, 2), MakeRef(FileX, 1)}},
- {C, {MakeRef(FileX, 5)}},
- {A, {MakeRef(FileX, 3)}},
- {C, {MakeRef(FileY, 4), MakeRef(FileX, 5)}},
+ {SymbolID("a"), {MakeRef(FileX, 3)}},
+ {SymbolID("b"), {MakeRef(FileY, 2), MakeRef(FileX, 1)}},
+ {SymbolID("c"), {MakeRef(FileY, 4), MakeRef(FileX, 5)}},
};
- auto Build = [&](bool WithDuplicateSymbol) {
- RefSlab::Builder Expected;
- RefSlab::BundleBuilder Actual;
- for (const auto &[ID, Refs] : Bundles) {
- if (!WithDuplicateSymbol && &Bundles.back().second == &Refs)
- continue;
- for (const Ref &R : Refs)
- Expected.insert(ID, R);
- Actual.insert(ID, Refs);
- }
- return std::make_pair(std::move(Expected).build(),
- std::move(Actual).build());
- };
- for (bool WithDuplicateSymbol : {false, true}) {
- auto [Expected, Actual] = Build(WithDuplicateSymbol);
- // Strings must be owned by the slabs.
- std::fill(FileX.begin(), FileX.end(), '?');
- std::fill(FileY.begin(), FileY.end(), '?');
- EXPECT_EQ(Expected.numRefs(), Actual.numRefs());
- ASSERT_EQ(Expected.size(), Actual.size());
- for (size_t I = 0; I < Expected.size(); ++I) {
- auto ExpectedIt = Expected.begin() + I, ActualIt = Actual.begin() + I;
- EXPECT_EQ(ExpectedIt->first, ActualIt->first);
- EXPECT_EQ(ExpectedIt->second, ActualIt->second);
- }
- FileX = "file:///x.cc";
- FileY = "file:///y.cc";
+ // BundleBuilder takes refs in RefSlab order.
+ llvm::sort(Bundles, llvm::less_first());
+ for (auto &[ID, Refs] : Bundles)
+ llvm::sort(Refs);
+
+ RefSlab::Builder ExpectedBuilder;
+ RefSlab::BundleBuilder ActualBuilder;
+ for (const auto &[ID, Refs] : Bundles) {
+ for (const Ref &R : Refs)
+ ExpectedBuilder.insert(ID, R);
+ ActualBuilder.insert(ID, Refs);
+ }
+ RefSlab Expected = std::move(ExpectedBuilder).build();
+ RefSlab Actual = std::move(ActualBuilder).build();
+ // Strings must be owned by the slabs.
+ std::fill(FileX.begin(), FileX.end(), '?');
+ std::fill(FileY.begin(), FileY.end(), '?');
+
+ EXPECT_EQ(Expected.numRefs(), Actual.numRefs());
+ ASSERT_EQ(Expected.size(), Actual.size());
+ for (size_t I = 0; I < Expected.size(); ++I) {
+ auto ExpectedIt = Expected.begin() + I, ActualIt = Actual.begin() + I;
+ EXPECT_EQ(ExpectedIt->first, ActualIt->first);
+ EXPECT_EQ(ExpectedIt->second, ActualIt->second);
}
}
TEST(RefSlabTest, BundleBuilderCopiesReusedBuffers) {
+ SymbolID First("a"), Second("b");
+ if (Second < First)
+ std::swap(First, Second);
// The same buffer holds a different file URI for each insert.
char Buffer[] = "file:///x.cc";
Ref R;
R.Location.FileURI = Buffer;
RefSlab::BundleBuilder Builder;
- Builder.insert(SymbolID("a"), {R});
+ Builder.insert(First, {R});
Buffer[8] = 'y';
- Builder.insert(SymbolID("b"), {R});
+ Builder.insert(Second, {R});
Buffer[8] = '?';
RefSlab Slab = std::move(Builder).build();
llvm::StringMap<SymbolID> FileToSymbol;
@@ -579,8 +575,8 @@ TEST(RefSlabTest, BundleBuilderCopiesReusedBuffers) {
for (const Ref &Ref : Refs)
FileToSymbol.try_emplace(Ref.Location.FileURI, ID);
EXPECT_EQ(FileToSymbol.size(), 2u);
- EXPECT_EQ(FileToSymbol.lookup("file:///x.cc"), SymbolID("a"));
- EXPECT_EQ(FileToSymbol.lookup("file:///y.cc"), SymbolID("b"));
+ EXPECT_EQ(FileToSymbol.lookup("file:///x.cc"), First);
+ EXPECT_EQ(FileToSymbol.lookup("file:///y.cc"), Second);
}
TEST(MergeIndexTest, IndexedFiles) {
diff --git a/clang-tools-extra/clangd/unittests/SerializationTests.cpp b/clang-tools-extra/clangd/unittests/SerializationTests.cpp
index cbe2a9726f8656..358fddba1a33ad 100644
--- a/clang-tools-extra/clangd/unittests/SerializationTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SerializationTests.cpp
@@ -543,6 +543,31 @@ TEST(SerializationTest, NoCrashOnBadArraySize) {
"malformed or truncated include uri");
}
+// Refs are written in RefSlab order: one bundle per symbol, sorted by symbol.
+TEST(SerializationTest, RejectsRepeatedRefsBundle) {
+ auto In = readIndexFile(YAML);
+ ASSERT_FALSE(!In) << In.takeError();
+ IndexFileOut Out(*In);
+ Out.Format = IndexFileFormat::RIFF;
+ std::string Serialized = llvm::to_string(Out);
+ ASSERT_FALSE(!readIndexFile(Serialized));
+
+ auto Parsed = riff::readFile(Serialized);
+ ASSERT_FALSE(!Parsed) << Parsed.takeError();
+ auto Refs = llvm::find_if(Parsed->Chunks, [](riff::Chunk C) {
+ return C.ID == riff::fourCC("refs");
+ });
+ ASSERT_NE(Refs, Parsed->Chunks.end());
+ // The fixture has refs for a single symbol; repeat its bundle.
+ std::string RepeatedRefs = (Refs->Data + Refs->Data).str();
+ Refs->Data = RepeatedRefs;
+
+ auto CorruptParsed = readIndexFile(llvm::to_string(*Parsed));
+ ASSERT_TRUE(!CorruptParsed);
+ EXPECT_EQ(llvm::toString(CorruptParsed.takeError()),
+ "malformed refs: not sorted or repeated symbol");
+}
+
// Check we detect invalid string table size size without allocating it first.
// If this detection fails, the test should allocate a huge array and crash.
TEST(SerializationTest, NoCrashOnBadStringTableSize) {
More information about the cfe-commits
mailing list