[clang-tools-extra] [clangd] Add background index format support to clangd-indexer (PR #175209)
Jason Williams via cfe-commits
cfe-commits at lists.llvm.org
Fri Jul 17 02:33:18 PDT 2026
https://github.com/jasonwilliams updated https://github.com/llvm/llvm-project/pull/175209
>From 86f4d8beccc66b57fe30f86b1d8b7a6bc2fbea7f Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Fri, 9 Jan 2026 17:23:51 +0000
Subject: [PATCH 1/8] [clangd] Add background index format support to
clangd-indexer
Add support for generating background index shards directly from clangd-indexer, enabling offline pre-indexing of projects for use with clangd's background index.
---
.../clangd/index/Serialization.h | 5 +-
.../clangd/indexer/IndexerMain.cpp | 205 ++++++++++++++++--
2 files changed, 191 insertions(+), 19 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h
index bf8e036afcb6c..1553e702a5881 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -35,8 +35,9 @@ namespace clang {
namespace clangd {
enum class IndexFileFormat {
- RIFF, // Versioned binary format, suitable for production use.
- YAML, // Human-readable format, suitable for experiments and debugging.
+ RIFF, // Versioned binary format, suitable for production use.
+ YAML, // Human-readable format, suitable for experiments and debugging.
+ BACKGROUND // Background index format, suitable for language server use.
};
// Holds the contents of an index file that was read.
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index 94db860f0b9b5..bf5eecaaaa6b4 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -12,6 +12,8 @@
#include "CompileCommands.h"
#include "Compiler.h"
+#include "GlobalCompilationDatabase.h"
+#include "index/Background.h"
#include "index/IndexAction.h"
#include "index/Merge.h"
#include "index/Ref.h"
@@ -30,13 +32,14 @@ namespace clang {
namespace clangd {
namespace {
-static llvm::cl::opt<IndexFileFormat>
- Format("format", llvm::cl::desc("Format of the index to be written"),
- llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml",
- "human-readable YAML format"),
- clEnumValN(IndexFileFormat::RIFF, "binary",
- "binary RIFF format")),
- llvm::cl::init(IndexFileFormat::RIFF));
+static llvm::cl::opt<IndexFileFormat> Format(
+ "format", llvm::cl::desc("Format of the index to be written"),
+ llvm::cl::values(
+ clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"),
+ clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"),
+ clEnumValN(IndexFileFormat::BACKGROUND, "background",
+ "background index format for language servers")),
+ llvm::cl::init(IndexFileFormat::RIFF));
static llvm::cl::list<std::string> QueryDriverGlobs{
"query-driver",
@@ -48,6 +51,16 @@ static llvm::cl::list<std::string> QueryDriverGlobs{
llvm::cl::CommaSeparated,
};
+static llvm::cl::opt<std::string> ProjectRoot{
+ "project-root",
+ llvm::cl::desc(
+ "Path to the project root for --format=background. "
+ "Determines where to store index shards. Shards are stored in "
+ "<project-root>/.cache/clangd/index/. "
+ "Defaults to current directory if not specified."),
+};
+
+// Action factory that merges all symbols into a single index (for YAML/RIFF).
class IndexActionFactory : public tooling::FrontendActionFactory {
public:
IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
@@ -123,6 +136,117 @@ class IndexActionFactory : public tooling::FrontendActionFactory {
RelationSlab::Builder Relations;
};
+// Action factory that writes per-file shards (for background index format).
+class BackgroundIndexActionFactory : public tooling::FrontendActionFactory {
+public:
+ BackgroundIndexActionFactory(BackgroundIndexStorage &Storage)
+ : Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()),
+ Refs(std::make_unique<RefSlab::Builder>()),
+ Relations(std::make_unique<RelationSlab::Builder>()) {}
+
+ std::unique_ptr<FrontendAction> create() override {
+ SymbolCollector::Options Opts;
+ Opts.CountReferences = true;
+ Opts.FileFilter = [&](const SourceManager &SM, FileID FID) {
+ const auto F = SM.getFileEntryRefForID(FID);
+ if (!F)
+ return false;
+ auto AbsPath = getCanonicalPath(*F, SM.getFileManager());
+ if (!AbsPath)
+ return false;
+ std::lock_guard<std::mutex> Lock(FilesMu);
+ return Files.insert(*AbsPath).second;
+ };
+ return createStaticIndexingAction(
+ Opts,
+ [&](SymbolSlab S) {
+ std::lock_guard<std::mutex> Lock(SymbolsMu);
+ for (const auto &Sym : S) {
+ if (const auto *Existing = Symbols->find(Sym.ID))
+ Symbols->insert(mergeSymbol(*Existing, Sym));
+ else
+ Symbols->insert(Sym);
+ }
+ },
+ [&](RefSlab S) {
+ std::lock_guard<std::mutex> Lock(RefsMu);
+ for (const auto &Sym : S) {
+ for (const auto &Ref : Sym.second)
+ Refs->insert(Sym.first, Ref);
+ }
+ },
+ [&](RelationSlab S) {
+ std::lock_guard<std::mutex> Lock(RelsMu);
+ for (const auto &R : S)
+ Relations->insert(R);
+ },
+ /*IncludeGraphCallback=*/nullptr);
+ }
+
+ bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+ FileManager *Files,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+ DiagnosticConsumer *DiagConsumer) override {
+ disableUnsupportedOptions(*Invocation);
+
+ // Get the main file path before running.
+ std::string MainFile;
+ if (!Invocation->getFrontendOpts().Inputs.empty())
+ MainFile = Invocation->getFrontendOpts().Inputs[0].getFile().str();
+
+ bool Success = tooling::FrontendActionFactory::runInvocation(
+ std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
+
+ // After processing, write a shard for this file.
+ if (Success && !MainFile.empty())
+ writeShardForFile(MainFile);
+
+ return Success;
+ }
+
+private:
+ void writeShardForFile(llvm::StringRef MainFile) {
+ IndexFileIn Data;
+ {
+ std::lock_guard<std::mutex> Lock(SymbolsMu);
+ Data.Symbols = std::move(*Symbols).build();
+ Symbols = std::make_unique<SymbolSlab::Builder>();
+ }
+ {
+ std::lock_guard<std::mutex> Lock(RefsMu);
+ Data.Refs = std::move(*Refs).build();
+ Refs = std::make_unique<RefSlab::Builder>();
+ }
+ {
+ std::lock_guard<std::mutex> Lock(RelsMu);
+ Data.Relations = std::move(*Relations).build();
+ Relations = std::make_unique<RelationSlab::Builder>();
+ }
+
+ IndexFileOut Out(Data);
+ Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format.
+
+ if (auto Err = Storage.storeShard(MainFile, Out)) {
+ elog("Failed to write shard for {0}: {1}", MainFile, std::move(Err));
+ } else {
+ std::lock_guard<std::mutex> Lock(FilesMu);
+ ++ShardsWritten;
+ log("Wrote shard for {0} ({1} total)", MainFile, ShardsWritten);
+ }
+ }
+
+ BackgroundIndexStorage &Storage;
+ std::mutex FilesMu;
+ llvm::StringSet<> Files;
+ unsigned ShardsWritten = 0;
+ std::mutex SymbolsMu;
+ std::unique_ptr<SymbolSlab::Builder> Symbols;
+ std::mutex RefsMu;
+ std::unique_ptr<RefSlab::Builder> Refs;
+ std::mutex RelsMu;
+ std::unique_ptr<RelationSlab::Builder> Relations;
+};
+
} // namespace
} // namespace clangd
} // namespace clang
@@ -141,6 +265,13 @@ int main(int argc, const char **argv) {
$ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex
+ Example usage for background index format (writes shards to disk):
+
+ $ clangd-indexer --format=background --executor=all-TUs build/
+
+ This writes index shards to .cache/clangd/index/ in the current directory.
+ Use --project-root to specify a different location for the shards.
+
Note: only symbols from header files will be indexed.
)";
@@ -152,23 +283,63 @@ int main(int argc, const char **argv) {
return 1;
}
- // Collect symbols found in each translation unit, merging as we go.
- clang::clangd::IndexFileIn Data;
auto Mangler = std::make_shared<clang::clangd::CommandMangler>(
clang::clangd::CommandMangler::detect());
Mangler->SystemIncludeExtractor = clang::clangd::getSystemIncludeExtractor(
static_cast<llvm::ArrayRef<std::string>>(
clang::clangd::QueryDriverGlobs));
+
+ auto Adjuster = clang::tooling::ArgumentsAdjuster(
+ [Mangler = std::move(Mangler)](const std::vector<std::string> &Args,
+ llvm::StringRef File) {
+ clang::tooling::CompileCommand Cmd;
+ Cmd.CommandLine = Args;
+ Mangler->operator()(Cmd, File);
+ return Cmd.CommandLine;
+ });
+
+ // Handle background index format separately - writes per-file shards.
+ if (clang::clangd::Format == clang::clangd::IndexFileFormat::BACKGROUND) {
+ // Default to current directory if --project-root not specified.
+ std::string Root = clang::clangd::ProjectRoot;
+ if (Root.empty()) {
+ llvm::SmallString<256> CurrentDir;
+ if (auto EC = llvm::sys::fs::current_path(CurrentDir)) {
+ llvm::errs() << "Error: Failed to get current directory: "
+ << EC.message() << "\n";
+ return 1;
+ }
+ Root = std::string(CurrentDir);
+ }
+
+ // Create storage factory for disk-backed index shards.
+ auto IndexStorageFactory =
+ clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory(
+ [Root](clang::clangd::PathRef) {
+ return clang::clangd::ProjectInfo{Root};
+ });
+
+ // Get storage for the project root.
+ clang::clangd::BackgroundIndexStorage *Storage = IndexStorageFactory(Root);
+
+ auto Err = Executor->get()->execute(
+ std::make_unique<clang::clangd::BackgroundIndexActionFactory>(*Storage),
+ std::move(Adjuster));
+ if (Err) {
+ clang::clangd::elog("{0}", std::move(Err));
+ return 1;
+ }
+
+ llvm::errs() << "Background index shards written to " << Root
+ << "/.cache/clangd/index/\n";
+ return 0;
+ }
+
+ // Standard mode: collect and merge symbols, then emit to stdout.
+ clang::clangd::IndexFileIn Data;
auto Err = Executor->get()->execute(
std::make_unique<clang::clangd::IndexActionFactory>(Data),
- clang::tooling::ArgumentsAdjuster(
- [Mangler = std::move(Mangler)](const std::vector<std::string> &Args,
- llvm::StringRef File) {
- clang::tooling::CompileCommand Cmd;
- Cmd.CommandLine = Args;
- Mangler->operator()(Cmd, File);
- return Cmd.CommandLine;
- }));
+ std::move(Adjuster));
if (Err) {
clang::clangd::elog("{0}", std::move(Err));
}
>From c6988120561d54a988f3ee268c771dacbad8053b Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Sun, 11 Jan 2026 15:01:40 +0000
Subject: [PATCH 2/8] handle background case for serialization
---
clang-tools-extra/clangd/index/Serialization.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index f03839599612c..10388b1948f43 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -686,6 +686,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
case IndexFileFormat::YAML:
writeYAML(O, OS);
break;
+ case IndexFileFormat::BACKGROUND:
+ llvm_unreachable("BACKGROUND format not supported for serialization");
}
return OS;
}
>From c4d335016379a89162785593d94a2de868b3f052 Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Fri, 23 Jan 2026 10:43:13 +0000
Subject: [PATCH 3/8] change name to sharded instead of background
---
.../clangd/index/Serialization.cpp | 4 ++--
.../clangd/index/Serialization.h | 6 ++---
.../clangd/indexer/IndexerMain.cpp | 22 +++++++++----------
3 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 10388b1948f43..fbb672f184dd2 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -686,8 +686,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
case IndexFileFormat::YAML:
writeYAML(O, OS);
break;
- case IndexFileFormat::BACKGROUND:
- llvm_unreachable("BACKGROUND format not supported for serialization");
+ case IndexFileFormat::SHARDED:
+ llvm_unreachable("SHARDED format not supported for serialization");
}
return OS;
}
diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h
index 1553e702a5881..d0939e051ecaf 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -35,9 +35,9 @@ namespace clang {
namespace clangd {
enum class IndexFileFormat {
- RIFF, // Versioned binary format, suitable for production use.
- YAML, // Human-readable format, suitable for experiments and debugging.
- BACKGROUND // Background index format, suitable for language server use.
+ RIFF, // Versioned binary format, suitable for production use.
+ YAML, // Human-readable format, suitable for experiments and debugging.
+ SHARDED // Sharded index format, suitable for language server use.
};
// Holds the contents of an index file that was read.
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index bf5eecaaaa6b4..64f610590d246 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -37,8 +37,8 @@ static llvm::cl::opt<IndexFileFormat> Format(
llvm::cl::values(
clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"),
clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"),
- clEnumValN(IndexFileFormat::BACKGROUND, "background",
- "background index format for language servers")),
+ clEnumValN(IndexFileFormat::SHARDED, "sharded",
+ "Sharded index format for language servers")),
llvm::cl::init(IndexFileFormat::RIFF));
static llvm::cl::list<std::string> QueryDriverGlobs{
@@ -54,7 +54,7 @@ static llvm::cl::list<std::string> QueryDriverGlobs{
static llvm::cl::opt<std::string> ProjectRoot{
"project-root",
llvm::cl::desc(
- "Path to the project root for --format=background. "
+ "Path to the project root for --format=sharded. "
"Determines where to store index shards. Shards are stored in "
"<project-root>/.cache/clangd/index/. "
"Defaults to current directory if not specified."),
@@ -136,10 +136,10 @@ class IndexActionFactory : public tooling::FrontendActionFactory {
RelationSlab::Builder Relations;
};
-// Action factory that writes per-file shards (for background index format).
-class BackgroundIndexActionFactory : public tooling::FrontendActionFactory {
+// Action factory that writes per-file shards (for sharded index format).
+class ShardedIndexActionFactory : public tooling::FrontendActionFactory {
public:
- BackgroundIndexActionFactory(BackgroundIndexStorage &Storage)
+ ShardedIndexActionFactory(BackgroundIndexStorage &Storage)
: Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()),
Refs(std::make_unique<RefSlab::Builder>()),
Relations(std::make_unique<RelationSlab::Builder>()) {}
@@ -265,9 +265,9 @@ int main(int argc, const char **argv) {
$ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex
- Example usage for background index format (writes shards to disk):
+ Example usage for sharded index format (writes shards to disk):
- $ clangd-indexer --format=background --executor=all-TUs build/
+ $ clangd-indexer --format=sharded --executor=all-TUs build/
This writes index shards to .cache/clangd/index/ in the current directory.
Use --project-root to specify a different location for the shards.
@@ -298,8 +298,8 @@ int main(int argc, const char **argv) {
return Cmd.CommandLine;
});
- // Handle background index format separately - writes per-file shards.
- if (clang::clangd::Format == clang::clangd::IndexFileFormat::BACKGROUND) {
+ // Handle sharded index format separately - writes per-file shards.
+ if (clang::clangd::Format == clang::clangd::IndexFileFormat::SHARDED) {
// Default to current directory if --project-root not specified.
std::string Root = clang::clangd::ProjectRoot;
if (Root.empty()) {
@@ -323,7 +323,7 @@ int main(int argc, const char **argv) {
clang::clangd::BackgroundIndexStorage *Storage = IndexStorageFactory(Root);
auto Err = Executor->get()->execute(
- std::make_unique<clang::clangd::BackgroundIndexActionFactory>(*Storage),
+ std::make_unique<clang::clangd::ShardedIndexActionFactory>(*Storage),
std::move(Adjuster));
if (Err) {
clang::clangd::elog("{0}", std::move(Err));
>From 15da5a92673205da23fa6a6823b7017a9663f15d Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Fri, 23 Jan 2026 10:50:59 +0000
Subject: [PATCH 4/8] create base class for both indexActionFactory and
ShardedIndexActionFactory
---
.../clangd/indexer/IndexerMain.cpp | 92 ++++++-------------
1 file changed, 27 insertions(+), 65 deletions(-)
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index 64f610590d246..c592551012336 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -60,10 +60,13 @@ static llvm::cl::opt<std::string> ProjectRoot{
"Defaults to current directory if not specified."),
};
-// Action factory that merges all symbols into a single index (for YAML/RIFF).
-class IndexActionFactory : public tooling::FrontendActionFactory {
+// Base class for index action factories that provides common symbol collection.
+class IndexActionFactoryBase : public tooling::FrontendActionFactory {
public:
- IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
+ IndexActionFactoryBase()
+ : Symbols(std::make_unique<SymbolSlab::Builder>()),
+ Refs(std::make_unique<RefSlab::Builder>()),
+ Relations(std::make_unique<RelationSlab::Builder>()) {}
std::unique_ptr<FrontendAction> create() override {
SymbolCollector::Options Opts;
@@ -116,72 +119,39 @@ class IndexActionFactory : public tooling::FrontendActionFactory {
std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
}
+protected:
+ std::mutex FilesMu;
+ llvm::StringSet<> Files;
+ std::mutex SymbolsMu;
+ std::unique_ptr<SymbolSlab::Builder> Symbols;
+ std::mutex RefsMu;
+ std::unique_ptr<RefSlab::Builder> Refs;
+ std::mutex RelsMu;
+ std::unique_ptr<RelationSlab::Builder> Relations;
+};
+
+// Action factory that merges all symbols into a single index (for YAML/RIFF).
+class IndexActionFactory : public IndexActionFactoryBase {
+public:
+ IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
+
// Awkward: we write the result in the destructor, because the executor
// takes ownership so it's the easiest way to get our data back out.
~IndexActionFactory() {
- Result.Symbols = std::move(Symbols).build();
- Result.Refs = std::move(Refs).build();
- Result.Relations = std::move(Relations).build();
+ Result.Symbols = std::move(*Symbols).build();
+ Result.Refs = std::move(*Refs).build();
+ Result.Relations = std::move(*Relations).build();
}
private:
IndexFileIn &Result;
- std::mutex FilesMu;
- llvm::StringSet<> Files;
- std::mutex SymbolsMu;
- SymbolSlab::Builder Symbols;
- std::mutex RefsMu;
- RefSlab::Builder Refs;
- std::mutex RelsMu;
- RelationSlab::Builder Relations;
};
// Action factory that writes per-file shards (for sharded index format).
-class ShardedIndexActionFactory : public tooling::FrontendActionFactory {
+class ShardedIndexActionFactory : public IndexActionFactoryBase {
public:
ShardedIndexActionFactory(BackgroundIndexStorage &Storage)
- : Storage(Storage), Symbols(std::make_unique<SymbolSlab::Builder>()),
- Refs(std::make_unique<RefSlab::Builder>()),
- Relations(std::make_unique<RelationSlab::Builder>()) {}
-
- std::unique_ptr<FrontendAction> create() override {
- SymbolCollector::Options Opts;
- Opts.CountReferences = true;
- Opts.FileFilter = [&](const SourceManager &SM, FileID FID) {
- const auto F = SM.getFileEntryRefForID(FID);
- if (!F)
- return false;
- auto AbsPath = getCanonicalPath(*F, SM.getFileManager());
- if (!AbsPath)
- return false;
- std::lock_guard<std::mutex> Lock(FilesMu);
- return Files.insert(*AbsPath).second;
- };
- return createStaticIndexingAction(
- Opts,
- [&](SymbolSlab S) {
- std::lock_guard<std::mutex> Lock(SymbolsMu);
- for (const auto &Sym : S) {
- if (const auto *Existing = Symbols->find(Sym.ID))
- Symbols->insert(mergeSymbol(*Existing, Sym));
- else
- Symbols->insert(Sym);
- }
- },
- [&](RefSlab S) {
- std::lock_guard<std::mutex> Lock(RefsMu);
- for (const auto &Sym : S) {
- for (const auto &Ref : Sym.second)
- Refs->insert(Sym.first, Ref);
- }
- },
- [&](RelationSlab S) {
- std::lock_guard<std::mutex> Lock(RelsMu);
- for (const auto &R : S)
- Relations->insert(R);
- },
- /*IncludeGraphCallback=*/nullptr);
- }
+ : Storage(Storage) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *Files,
@@ -236,15 +206,7 @@ class ShardedIndexActionFactory : public tooling::FrontendActionFactory {
}
BackgroundIndexStorage &Storage;
- std::mutex FilesMu;
- llvm::StringSet<> Files;
unsigned ShardsWritten = 0;
- std::mutex SymbolsMu;
- std::unique_ptr<SymbolSlab::Builder> Symbols;
- std::mutex RefsMu;
- std::unique_ptr<RefSlab::Builder> Refs;
- std::mutex RelsMu;
- std::unique_ptr<RelationSlab::Builder> Relations;
};
} // namespace
>From e88c938833277ef44ceb416e3db63e3f35a7eb31 Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Mon, 26 Jan 2026 11:32:44 +0000
Subject: [PATCH 5/8] Change sharded from format to index-type instead
---
.../clangd/index/Serialization.cpp | 2 -
.../clangd/index/Serialization.h | 5 +--
.../clangd/indexer/IndexerMain.cpp | 37 ++++++++++++-------
3 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index fbb672f184dd2..f03839599612c 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -686,8 +686,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
case IndexFileFormat::YAML:
writeYAML(O, OS);
break;
- case IndexFileFormat::SHARDED:
- llvm_unreachable("SHARDED format not supported for serialization");
}
return OS;
}
diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h
index d0939e051ecaf..bf8e036afcb6c 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -35,9 +35,8 @@ namespace clang {
namespace clangd {
enum class IndexFileFormat {
- RIFF, // Versioned binary format, suitable for production use.
- YAML, // Human-readable format, suitable for experiments and debugging.
- SHARDED // Sharded index format, suitable for language server use.
+ RIFF, // Versioned binary format, suitable for production use.
+ YAML, // Human-readable format, suitable for experiments and debugging.
};
// Holds the contents of an index file that was read.
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index c592551012336..36c66dc3356ef 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -30,16 +30,27 @@
namespace clang {
namespace clangd {
+
+enum class IndexOutputMode { Monolithic, Sharded };
+
namespace {
-static llvm::cl::opt<IndexFileFormat> Format(
- "format", llvm::cl::desc("Format of the index to be written"),
+static llvm::cl::opt<IndexFileFormat>
+ Format("format", llvm::cl::desc("Format of the index to be written"),
+ llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml",
+ "human-readable YAML format"),
+ clEnumValN(IndexFileFormat::RIFF, "binary",
+ "binary RIFF format")),
+ llvm::cl::init(IndexFileFormat::RIFF));
+
+static llvm::cl::opt<IndexOutputMode> OutputMode(
+ "index-type", llvm::cl::desc("Type of index output"),
llvm::cl::values(
- clEnumValN(IndexFileFormat::YAML, "yaml", "human-readable YAML format"),
- clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format"),
- clEnumValN(IndexFileFormat::SHARDED, "sharded",
- "Sharded index format for language servers")),
- llvm::cl::init(IndexFileFormat::RIFF));
+ clEnumValN(IndexOutputMode::Monolithic, "monolithic",
+ "Single merged index file written to stdout (default)"),
+ clEnumValN(IndexOutputMode::Sharded, "sharded",
+ "Per-file shards written to disk")),
+ llvm::cl::init(IndexOutputMode::Monolithic));
static llvm::cl::list<std::string> QueryDriverGlobs{
"query-driver",
@@ -54,7 +65,7 @@ static llvm::cl::list<std::string> QueryDriverGlobs{
static llvm::cl::opt<std::string> ProjectRoot{
"project-root",
llvm::cl::desc(
- "Path to the project root for --format=sharded. "
+ "Path to the project root for --index-type=sharded. "
"Determines where to store index shards. Shards are stored in "
"<project-root>/.cache/clangd/index/. "
"Defaults to current directory if not specified."),
@@ -227,9 +238,9 @@ int main(int argc, const char **argv) {
$ clangd-indexer File1.cpp File2.cpp ... FileN.cpp > clangd.dex
- Example usage for sharded index format (writes shards to disk):
+ Example usage for sharded index (writes shards to disk):
- $ clangd-indexer --format=sharded --executor=all-TUs build/
+ $ clangd-indexer --index-type=sharded --executor=all-TUs build/
This writes index shards to .cache/clangd/index/ in the current directory.
Use --project-root to specify a different location for the shards.
@@ -260,8 +271,8 @@ int main(int argc, const char **argv) {
return Cmd.CommandLine;
});
- // Handle sharded index format separately - writes per-file shards.
- if (clang::clangd::Format == clang::clangd::IndexFileFormat::SHARDED) {
+ // Handle sharded index type separately - writes per-file shards.
+ if (clang::clangd::OutputMode == clang::clangd::IndexOutputMode::Sharded) {
// Default to current directory if --project-root not specified.
std::string Root = clang::clangd::ProjectRoot;
if (Root.empty()) {
@@ -292,7 +303,7 @@ int main(int argc, const char **argv) {
return 1;
}
- llvm::errs() << "Background index shards written to " << Root
+ llvm::errs() << "Index shards written to " << Root
<< "/.cache/clangd/index/\n";
return 0;
}
>From efea3c23424e5dc9f1e9c787467583b1fad15a5e Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Mon, 26 Jan 2026 17:00:25 +0000
Subject: [PATCH 6/8] Match how clangd's background index works
---
.../clangd/indexer/IndexerMain.cpp | 107 ++++++++++++++++--
1 file changed, 96 insertions(+), 11 deletions(-)
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index 36c66dc3356ef..7bdd2f68e609d 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -14,6 +14,7 @@
#include "Compiler.h"
#include "GlobalCompilationDatabase.h"
#include "index/Background.h"
+#include "index/FileIndex.h"
#include "index/IndexAction.h"
#include "index/Merge.h"
#include "index/Ref.h"
@@ -21,6 +22,7 @@
#include "index/Symbol.h"
#include "index/SymbolCollector.h"
#include "support/Logger.h"
+#include "URI.h"
#include "clang/Tooling/ArgumentsAdjusters.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
@@ -164,6 +166,54 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase {
ShardedIndexActionFactory(BackgroundIndexStorage &Storage)
: Storage(Storage) {}
+ std::unique_ptr<FrontendAction> create() override {
+ SymbolCollector::Options Opts;
+ Opts.CountReferences = true;
+ Opts.FileFilter = [&](const SourceManager &SM, FileID FID) {
+ const auto F = SM.getFileEntryRefForID(FID);
+ if (!F)
+ return false; // Skip invalid files.
+ auto AbsPath = getCanonicalPath(*F, SM.getFileManager());
+ if (!AbsPath)
+ return false; // Skip files without absolute path.
+ std::lock_guard<std::mutex> Lock(FilesMu);
+ return Files.insert(*AbsPath).second; // Skip already processed files.
+ };
+ return createStaticIndexingAction(
+ Opts,
+ [&](SymbolSlab S) {
+ // Merge as we go.
+ std::lock_guard<std::mutex> Lock(SymbolsMu);
+ for (const auto &Sym : S) {
+ if (const auto *Existing = Symbols->find(Sym.ID))
+ Symbols->insert(mergeSymbol(*Existing, Sym));
+ else
+ Symbols->insert(Sym);
+ }
+ },
+ [&](RefSlab S) {
+ std::lock_guard<std::mutex> Lock(RefsMu);
+ for (const auto &Sym : S) {
+ // Deduplication happens during insertion.
+ for (const auto &Ref : Sym.second)
+ Refs->insert(Sym.first, Ref);
+ }
+ },
+ [&](RelationSlab S) {
+ std::lock_guard<std::mutex> Lock(RelsMu);
+ for (const auto &R : S) {
+ Relations->insert(R);
+ }
+ },
+ [&](IncludeGraph IG) {
+ std::lock_guard<std::mutex> Lock(SourcesMu);
+ for (auto &Entry : IG) {
+ // Merge include graphs from different TUs.
+ Sources.try_emplace(Entry.first(), Entry.second);
+ }
+ });
+ }
+
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *Files,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
@@ -178,15 +228,16 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase {
bool Success = tooling::FrontendActionFactory::runInvocation(
std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
- // After processing, write a shard for this file.
+ // After processing, write shards for all files in this TU.
if (Success && !MainFile.empty())
- writeShardForFile(MainFile);
+ writeShardsForTU(MainFile);
return Success;
}
private:
- void writeShardForFile(llvm::StringRef MainFile) {
+ void writeShardsForTU(llvm::StringRef MainFile) {
+ // Build the complete index data for this TU.
IndexFileIn Data;
{
std::lock_guard<std::mutex> Lock(SymbolsMu);
@@ -203,20 +254,54 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase {
Data.Relations = std::move(*Relations).build();
Relations = std::make_unique<RelationSlab::Builder>();
}
+ {
+ std::lock_guard<std::mutex> Lock(SourcesMu);
+ Data.Sources = std::move(Sources);
+ Sources.clear();
+ }
- IndexFileOut Out(Data);
- Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format.
+ // Shard the index data per-file.
+ FileShardedIndex ShardedIndex(std::move(Data));
- if (auto Err = Storage.storeShard(MainFile, Out)) {
- elog("Failed to write shard for {0}: {1}", MainFile, std::move(Err));
- } else {
- std::lock_guard<std::mutex> Lock(FilesMu);
- ++ShardsWritten;
- log("Wrote shard for {0} ({1} total)", MainFile, ShardsWritten);
+ // Write a shard for each file.
+ unsigned TUShardsWritten = 0;
+ for (llvm::StringRef Uri : ShardedIndex.getAllSources()) {
+ auto Shard = ShardedIndex.getShard(Uri);
+ if (!Shard) {
+ elog("Failed to get shard for {0}", Uri);
+ continue;
+ }
+
+ // Resolve URI to absolute path.
+ auto AbsPath = URI::resolve(Uri, MainFile);
+ if (!AbsPath) {
+ elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError());
+ continue;
+ }
+
+ // Only store command line for the main file.
+ if (*AbsPath != MainFile)
+ Shard->Cmd.reset();
+
+ IndexFileOut Out(*Shard);
+ Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format.
+
+ if (auto Err = Storage.storeShard(*AbsPath, Out)) {
+ elog("Failed to write shard for {0}: {1}", *AbsPath, std::move(Err));
+ } else {
+ ++TUShardsWritten;
+ }
}
+
+ std::lock_guard<std::mutex> Lock(FilesMu);
+ ShardsWritten += TUShardsWritten;
+ log("Wrote {0} shards for TU {1} ({2} total)", TUShardsWritten, MainFile,
+ ShardsWritten);
}
BackgroundIndexStorage &Storage;
+ std::mutex SourcesMu;
+ IncludeGraph Sources;
unsigned ShardsWritten = 0;
};
>From f65767524a762ca1bc225ba7e8fede5c88cce5bf Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Mon, 26 Jan 2026 18:41:16 +0000
Subject: [PATCH 7/8] fixup formatting
---
clang-tools-extra/clangd/indexer/IndexerMain.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index 7bdd2f68e609d..98902af73ebdb 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -13,6 +13,7 @@
#include "CompileCommands.h"
#include "Compiler.h"
#include "GlobalCompilationDatabase.h"
+#include "URI.h"
#include "index/Background.h"
#include "index/FileIndex.h"
#include "index/IndexAction.h"
@@ -22,7 +23,6 @@
#include "index/Symbol.h"
#include "index/SymbolCollector.h"
#include "support/Logger.h"
-#include "URI.h"
#include "clang/Tooling/ArgumentsAdjusters.h"
#include "clang/Tooling/Execution.h"
#include "clang/Tooling/Tooling.h"
>From 9d6c7f78ee7eb439eec78d66eff1765909701846 Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams at gmail.com>
Date: Mon, 13 Jul 2026 19:22:45 +0100
Subject: [PATCH 8/8] =?UTF-8?q?-=20Removed=20IndexActionFactoryBase=20?=
=?UTF-8?q?=E2=80=94=20the=20shared=20base=20class=20is=20gone.=20=20=20In?=
=?UTF-8?q?dexActionFactory=20is=20now=20a=20standalone=20class=20(restore?=
=?UTF-8?q?d=20to=20its=20pre-PR=20=20=20state)=20with=20direct=20member?=
=?UTF-8?q?=20builders=20(SymbolSlab::Builder=20Symbols,=20etc.)=20=20=20r?=
=?UTF-8?q?ather=20than=20unique=5Fptrs.=20-=20Completely=20rewrote=20Shar?=
=?UTF-8?q?dedIndexActionFactory:=20=20=20-=20No=20more=20merging=20betwee?=
=?UTF-8?q?n=20TUs,=20each=20IndexFileIn=20from=20the=20=20=20=20=20create?=
=?UTF-8?q?StaticIndexingAction=20callback=20is=20passed=20directly=20to?=
=?UTF-8?q?=20=20=20=20=20FileShardedIndex=20=20=20-=20No=20SymbolSlab::Bu?=
=?UTF-8?q?ilder=20/=20RefSlab::Builder=20/=20RelationSlab::Builder=20=20?=
=?UTF-8?q?=20=20=20objects,=20slabs=20go=20straight=20from=20the=20callba?=
=?UTF-8?q?ck=20into=20FileShardedIndex=20=20=20-=20Shards=20are=20written?=
=?UTF-8?q?=20directly=20from=20the=20createStaticIndexingAction=20=20=20?=
=?UTF-8?q?=20=20callback=20(no=20more=20runInvocation=20override=20needed?=
=?UTF-8?q?=20for=20the=20=20=20=20=20shard-writing=20logic)=20=20=20-=20N?=
=?UTF-8?q?o=20longer=20overrides=20create()=20with=20the=20(non-existent)?=
=?UTF-8?q?=204-callback=20form=20-=20Added=20header=20shard=20deduplicati?=
=?UTF-8?q?on=20=E2=80=94=20ShardVersions=20(a=20=20=20StringMap<ShardVers?=
=?UTF-8?q?ion>)=20tracks=20the=20content=20digest=20of=20each=20shard=20w?=
=?UTF-8?q?e've=20=20=20written.=20Before=20each=20TU=20is=20indexed,=20cr?=
=?UTF-8?q?eate()=20snapshots=20this=20map.=20=20=20writeShards()=20then?=
=?UTF-8?q?=20only=20writes=20shards=20for=20files=20whose=20digest=20diff?=
=?UTF-8?q?ers=20=20=20from=20the=20snapshot,=20mirroring=20exactly=20how?=
=?UTF-8?q?=20BackgroundIndex::update()=20=20=20avoids=20re-writing=20unch?=
=?UTF-8?q?anged=20header=20shards.=20-=20Moved=20ShardVersion=20to=20name?=
=?UTF-8?q?space=20scope=20as=20requested=20by=20the=20reviewer.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../clangd/indexer/IndexerMain.cpp | 216 +++++++-----------
1 file changed, 84 insertions(+), 132 deletions(-)
diff --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index 98902af73ebdb..d41b68a427c88 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -73,13 +73,17 @@ static llvm::cl::opt<std::string> ProjectRoot{
"Defaults to current directory if not specified."),
};
-// Base class for index action factories that provides common symbol collection.
-class IndexActionFactoryBase : public tooling::FrontendActionFactory {
+// Tracks the content digest of the last shard written for a given file.
+// Used to avoid writing duplicate shards when multiple TUs include the same
+// header.
+struct ShardVersion {
+ FileDigest Digest{{0}};
+};
+
+// Action factory that merges all symbols into a single index (for YAML/RIFF).
+class IndexActionFactory : public tooling::FrontendActionFactory {
public:
- IndexActionFactoryBase()
- : Symbols(std::make_unique<SymbolSlab::Builder>()),
- Refs(std::make_unique<RefSlab::Builder>()),
- Relations(std::make_unique<RelationSlab::Builder>()) {}
+ IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
std::unique_ptr<FrontendAction> create() override {
SymbolCollector::Options Opts;
@@ -94,11 +98,11 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory {
std::lock_guard<std::mutex> Lock(FilesMu);
return Files.insert(*AbsPath).second; // Skip already processed files.
};
- return createStaticIndexingAction(Opts, [&](IndexFileIn Result) {
+ return createStaticIndexingAction(Opts, [&](IndexFileIn TUResult) {
{
// Merge as we go.
std::lock_guard<std::mutex> Lock(SymbolsMu);
- for (const auto &Sym : *Result.Symbols) {
+ for (const auto &Sym : *TUResult.Symbols) {
if (const auto *Existing = Symbols.find(Sym.ID))
Symbols.insert(mergeSymbol(*Existing, Sym));
else
@@ -107,7 +111,7 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory {
}
{
std::lock_guard<std::mutex> Lock(RefsMu);
- for (const auto &Sym : *Result.Refs) {
+ for (const auto &Sym : *TUResult.Refs) {
// Deduplication happens during insertion.
for (const auto &Ref : Sym.second)
Refs.insert(Sym.first, Ref);
@@ -115,11 +119,11 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory {
}
{
std::lock_guard<std::mutex> Lock(RelsMu);
- for (const auto &R : *Result.Relations) {
+ for (const auto &R : *TUResult.Relations) {
Relations.insert(R);
}
}
- // FIXME: Handle Result.Sources?
+ // FIXME: Handle TUResult.Sources?
});
}
@@ -132,85 +136,50 @@ class IndexActionFactoryBase : public tooling::FrontendActionFactory {
std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
}
-protected:
- std::mutex FilesMu;
- llvm::StringSet<> Files;
- std::mutex SymbolsMu;
- std::unique_ptr<SymbolSlab::Builder> Symbols;
- std::mutex RefsMu;
- std::unique_ptr<RefSlab::Builder> Refs;
- std::mutex RelsMu;
- std::unique_ptr<RelationSlab::Builder> Relations;
-};
-
-// Action factory that merges all symbols into a single index (for YAML/RIFF).
-class IndexActionFactory : public IndexActionFactoryBase {
-public:
- IndexActionFactory(IndexFileIn &Result) : Result(Result) {}
-
// Awkward: we write the result in the destructor, because the executor
// takes ownership so it's the easiest way to get our data back out.
~IndexActionFactory() {
- Result.Symbols = std::move(*Symbols).build();
- Result.Refs = std::move(*Refs).build();
- Result.Relations = std::move(*Relations).build();
+ Result.Symbols = std::move(Symbols).build();
+ Result.Refs = std::move(Refs).build();
+ Result.Relations = std::move(Relations).build();
}
private:
IndexFileIn &Result;
+ std::mutex FilesMu;
+ llvm::StringSet<> Files;
+ std::mutex SymbolsMu;
+ SymbolSlab::Builder Symbols;
+ std::mutex RefsMu;
+ RefSlab::Builder Refs;
+ std::mutex RelsMu;
+ RelationSlab::Builder Relations;
};
// Action factory that writes per-file shards (for sharded index format).
-class ShardedIndexActionFactory : public IndexActionFactoryBase {
+// Each TU's index data is sharded independently — no merging across TUs.
+// Header shards are deduplicated: if a header's content hasn't changed since
+// the last time we wrote its shard, we skip writing it again.
+class ShardedIndexActionFactory : public tooling::FrontendActionFactory {
public:
ShardedIndexActionFactory(BackgroundIndexStorage &Storage)
: Storage(Storage) {}
std::unique_ptr<FrontendAction> create() override {
+ // Snapshot the current shard versions so the callback can check staleness
+ // without holding the lock during indexing.
+ llvm::StringMap<ShardVersion> Snapshot;
+ {
+ std::lock_guard<std::mutex> Lock(ShardVersionsMu);
+ Snapshot = ShardVersions;
+ }
+
SymbolCollector::Options Opts;
Opts.CountReferences = true;
- Opts.FileFilter = [&](const SourceManager &SM, FileID FID) {
- const auto F = SM.getFileEntryRefForID(FID);
- if (!F)
- return false; // Skip invalid files.
- auto AbsPath = getCanonicalPath(*F, SM.getFileManager());
- if (!AbsPath)
- return false; // Skip files without absolute path.
- std::lock_guard<std::mutex> Lock(FilesMu);
- return Files.insert(*AbsPath).second; // Skip already processed files.
- };
+
return createStaticIndexingAction(
- Opts,
- [&](SymbolSlab S) {
- // Merge as we go.
- std::lock_guard<std::mutex> Lock(SymbolsMu);
- for (const auto &Sym : S) {
- if (const auto *Existing = Symbols->find(Sym.ID))
- Symbols->insert(mergeSymbol(*Existing, Sym));
- else
- Symbols->insert(Sym);
- }
- },
- [&](RefSlab S) {
- std::lock_guard<std::mutex> Lock(RefsMu);
- for (const auto &Sym : S) {
- // Deduplication happens during insertion.
- for (const auto &Ref : Sym.second)
- Refs->insert(Sym.first, Ref);
- }
- },
- [&](RelationSlab S) {
- std::lock_guard<std::mutex> Lock(RelsMu);
- for (const auto &R : S) {
- Relations->insert(R);
- }
- },
- [&](IncludeGraph IG) {
- std::lock_guard<std::mutex> Lock(SourcesMu);
- for (auto &Entry : IG) {
- // Merge include graphs from different TUs.
- Sources.try_emplace(Entry.first(), Entry.second);
- }
+ Opts, [this, Snapshot](IndexFileIn Index) {
+ writeShards(std::move(Index), Snapshot);
});
}
@@ -219,90 +188,73 @@ class ShardedIndexActionFactory : public IndexActionFactoryBase {
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer) override {
disableUnsupportedOptions(*Invocation);
-
- // Get the main file path before running.
- std::string MainFile;
- if (!Invocation->getFrontendOpts().Inputs.empty())
- MainFile = Invocation->getFrontendOpts().Inputs[0].getFile().str();
-
- bool Success = tooling::FrontendActionFactory::runInvocation(
+ return tooling::FrontendActionFactory::runInvocation(
std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
-
- // After processing, write shards for all files in this TU.
- if (Success && !MainFile.empty())
- writeShardsForTU(MainFile);
-
- return Success;
}
private:
- void writeShardsForTU(llvm::StringRef MainFile) {
- // Build the complete index data for this TU.
- IndexFileIn Data;
- {
- std::lock_guard<std::mutex> Lock(SymbolsMu);
- Data.Symbols = std::move(*Symbols).build();
- Symbols = std::make_unique<SymbolSlab::Builder>();
- }
- {
- std::lock_guard<std::mutex> Lock(RefsMu);
- Data.Refs = std::move(*Refs).build();
- Refs = std::make_unique<RefSlab::Builder>();
- }
- {
- std::lock_guard<std::mutex> Lock(RelsMu);
- Data.Relations = std::move(*Relations).build();
- Relations = std::make_unique<RelationSlab::Builder>();
+ void writeShards(IndexFileIn Index,
+ const llvm::StringMap<ShardVersion> &Snapshot) {
+ if (!Index.Sources)
+ return;
+
+ // Find the URI of the main file (the TU root). Store as owned string
+ // because we move Index below, which would invalidate any StringRef into it.
+ std::string MainUri;
+ for (const auto &[Uri, Node] : *Index.Sources) {
+ if (Node.Flags & IncludeGraphNode::SourceFlag::IsTU) {
+ MainUri = Uri.str();
+ break;
+ }
}
- {
- std::lock_guard<std::mutex> Lock(SourcesMu);
- Data.Sources = std::move(Sources);
- Sources.clear();
+
+ // Collect files that need updated shards, based on content digest.
+ // Files whose shard was already written with the same digest are skipped.
+ llvm::StringMap<std::pair<Path, FileDigest>> FilesToUpdate;
+ for (const auto &[Uri, Node] : *Index.Sources) {
+ auto AbsPath = URI::resolve(Uri, MainUri);
+ if (!AbsPath) {
+ elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError());
+ continue;
+ }
+ auto DigestIt = Snapshot.find(*AbsPath);
+ if (DigestIt == Snapshot.end() || DigestIt->second.Digest != Node.Digest)
+ FilesToUpdate[Uri] = {std::move(*AbsPath), Node.Digest};
}
- // Shard the index data per-file.
- FileShardedIndex ShardedIndex(std::move(Data));
+ // Shard the index data by file.
+ FileShardedIndex ShardedIndex(std::move(Index));
- // Write a shard for each file.
- unsigned TUShardsWritten = 0;
- for (llvm::StringRef Uri : ShardedIndex.getAllSources()) {
+ unsigned Written = 0;
+ for (const auto &[Uri, PathAndDigest] : FilesToUpdate) {
auto Shard = ShardedIndex.getShard(Uri);
if (!Shard) {
elog("Failed to get shard for {0}", Uri);
continue;
}
+ PathRef Path = PathAndDigest.first;
- // Resolve URI to absolute path.
- auto AbsPath = URI::resolve(Uri, MainFile);
- if (!AbsPath) {
- elog("Failed to resolve URI {0}: {1}", Uri, AbsPath.takeError());
- continue;
- }
-
- // Only store command line for the main file.
- if (*AbsPath != MainFile)
+ // Command line is only meaningful for the TU's main file.
+ if (Uri != MainUri)
Shard->Cmd.reset();
IndexFileOut Out(*Shard);
- Out.Format = IndexFileFormat::RIFF; // Shards use RIFF format.
-
- if (auto Err = Storage.storeShard(*AbsPath, Out)) {
- elog("Failed to write shard for {0}: {1}", *AbsPath, std::move(Err));
+ Out.Format = IndexFileFormat::RIFF;
+ if (auto Err = Storage.storeShard(Path, Out)) {
+ elog("Failed to write shard for {0}: {1}", Path, std::move(Err));
} else {
- ++TUShardsWritten;
+ std::lock_guard<std::mutex> Lock(ShardVersionsMu);
+ ShardVersions[Path].Digest = PathAndDigest.second;
+ ++Written;
}
}
- std::lock_guard<std::mutex> Lock(FilesMu);
- ShardsWritten += TUShardsWritten;
- log("Wrote {0} shards for TU {1} ({2} total)", TUShardsWritten, MainFile,
- ShardsWritten);
+ log("Wrote {0} shards for TU {1}", Written, MainUri);
}
BackgroundIndexStorage &Storage;
- std::mutex SourcesMu;
- IncludeGraph Sources;
- unsigned ShardsWritten = 0;
+ std::mutex ShardVersionsMu;
+ llvm::StringMap<ShardVersion> ShardVersions;
};
} // namespace
More information about the cfe-commits
mailing list