[clang-tools-extra] 067ffbf - [clangd] Introduce ProjectAwareIndex
Kadir Cetinkaya via cfe-commits
cfe-commits at lists.llvm.org
Sun Nov 22 12:13:20 PST 2020
Author: Kadir Cetinkaya
Date: 2020-11-22T20:59:37+01:00
New Revision: 067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76
URL: https://github.com/llvm/llvm-project/commit/067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76
DIFF: https://github.com/llvm/llvm-project/commit/067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76.diff
LOG: [clangd] Introduce ProjectAwareIndex
An index implementation that can dispatch to a variety of indexes
depending on the file path. Enables clangd to work with multiple indexes in the
same instance, configured via config files.
Depends on D90749, D90746
Differential Revision: https://reviews.llvm.org/D90750
Added:
clang-tools-extra/clangd/index/ProjectAware.cpp
clang-tools-extra/clangd/index/ProjectAware.h
clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
Modified:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/Config.h
clang-tools-extra/clangd/unittests/CMakeLists.txt
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 3fd110e9e135..d02a5cf3f2ec 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -97,6 +97,7 @@ add_clang_library(clangDaemon
index/IndexAction.cpp
index/MemIndex.cpp
index/Merge.cpp
+ index/ProjectAware.cpp
index/Ref.cpp
index/Relation.cpp
index/Serialization.cpp
@@ -150,6 +151,7 @@ target_link_libraries(clangDaemon
clangTidy
${ALL_CLANG_TIDY_CHECKS}
+ clangdRemoteIndex
clangdSupport
)
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index 318220364e6e..79e94ef6fe37 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -97,4 +97,24 @@ struct Config {
} // namespace clangd
} // namespace clang
+namespace llvm {
+template <> struct DenseMapInfo<clang::clangd::Config::ExternalIndexSpec> {
+ using ExternalIndexSpec = clang::clangd::Config::ExternalIndexSpec;
+ static inline ExternalIndexSpec getEmptyKey() {
+ return {ExternalIndexSpec::File, "", ""};
+ }
+ static inline ExternalIndexSpec getTombstoneKey() {
+ return {ExternalIndexSpec::File, "TOMB", "STONE"};
+ }
+ static unsigned getHashValue(const ExternalIndexSpec &Val) {
+ return llvm::hash_combine(Val.Kind, Val.Location, Val.MountPoint);
+ }
+ static bool isEqual(const ExternalIndexSpec &LHS,
+ const ExternalIndexSpec &RHS) {
+ return std::tie(LHS.Kind, LHS.Location, LHS.MountPoint) ==
+ std::tie(RHS.Kind, RHS.Location, RHS.MountPoint);
+ }
+};
+} // namespace llvm
+
#endif
diff --git a/clang-tools-extra/clangd/index/ProjectAware.cpp b/clang-tools-extra/clangd/index/ProjectAware.cpp
new file mode 100644
index 000000000000..58685fd14bf5
--- /dev/null
+++ b/clang-tools-extra/clangd/index/ProjectAware.cpp
@@ -0,0 +1,156 @@
+//===--- ProjectAware.h ------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProjectAware.h"
+#include "Config.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/Merge.h"
+#include "index/Ref.h"
+#include "index/Serialization.h"
+#include "index/Symbol.h"
+#include "index/SymbolID.h"
+#include "index/remote/Client.h"
+#include "support/Logger.h"
+#include "support/Threading.h"
+#include "support/Trace.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <map>
+#include <memory>
+#include <mutex>
+#include <tuple>
+
+namespace clang {
+namespace clangd {
+namespace {
+class ProjectAwareIndex : public SymbolIndex {
+public:
+ size_t estimateMemoryUsage() const override;
+
+ /// Only queries the associated index with the current context.
+ void lookup(const LookupRequest &Req,
+ llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+ /// Query all indexes while prioritizing the associated one (if any).
+ bool refs(const RefsRequest &Req,
+ llvm::function_ref<void(const Ref &)> Callback) const override;
+
+ /// Queries only the associates index when Req.RestrictForCodeCompletion is
+ /// set, otherwise queries all.
+ bool
+ fuzzyFind(const FuzzyFindRequest &Req,
+ llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+ /// Query all indexes while prioritizing the associated one (if any).
+ void relations(const RelationsRequest &Req,
+ llvm::function_ref<void(const SymbolID &, const Symbol &)>
+ Callback) const override;
+
+ ProjectAwareIndex(IndexFactory Gen) : Gen(std::move(Gen)) {}
+
+private:
+ // Returns the index associated with current context, if any.
+ SymbolIndex *getIndex() const;
+
+ // Storage for all the external indexes.
+ mutable std::mutex Mu;
+ mutable llvm::DenseMap<Config::ExternalIndexSpec,
+ std::unique_ptr<SymbolIndex>>
+ IndexForSpec;
+ mutable AsyncTaskRunner Tasks;
+
+ const IndexFactory Gen;
+};
+
+size_t ProjectAwareIndex::estimateMemoryUsage() const {
+ size_t Total = 0;
+ std::lock_guard<std::mutex> Lock(Mu);
+ for (auto &Entry : IndexForSpec)
+ Total += Entry.second->estimateMemoryUsage();
+ return Total;
+}
+
+void ProjectAwareIndex::lookup(
+ const LookupRequest &Req,
+ llvm::function_ref<void(const Symbol &)> Callback) const {
+ trace::Span Tracer("ProjectAwareIndex::lookup");
+ if (auto *Idx = getIndex())
+ Idx->lookup(Req, Callback);
+}
+
+bool ProjectAwareIndex::refs(
+ const RefsRequest &Req,
+ llvm::function_ref<void(const Ref &)> Callback) const {
+ trace::Span Tracer("ProjectAwareIndex::refs");
+ if (auto *Idx = getIndex())
+ return Idx->refs(Req, Callback);
+ return false;
+}
+
+bool ProjectAwareIndex::fuzzyFind(
+ const FuzzyFindRequest &Req,
+ llvm::function_ref<void(const Symbol &)> Callback) const {
+ trace::Span Tracer("ProjectAwareIndex::fuzzyFind");
+ if (auto *Idx = getIndex())
+ return Idx->fuzzyFind(Req, Callback);
+ return false;
+}
+
+void ProjectAwareIndex::relations(
+ const RelationsRequest &Req,
+ llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
+ trace::Span Tracer("ProjectAwareIndex::relations");
+ if (auto *Idx = getIndex())
+ return Idx->relations(Req, Callback);
+}
+
+SymbolIndex *ProjectAwareIndex::getIndex() const {
+ const auto &C = Config::current();
+ if (!C.Index.External)
+ return nullptr;
+ const auto &External = *C.Index.External;
+ std::lock_guard<std::mutex> Lock(Mu);
+ auto Entry = IndexForSpec.try_emplace(External, nullptr);
+ if (Entry.second)
+ Entry.first->getSecond() = Gen(External, Tasks);
+ return Entry.first->second.get();
+}
+
+std::unique_ptr<SymbolIndex>
+defaultFactory(const Config::ExternalIndexSpec &External,
+ AsyncTaskRunner &Tasks) {
+ switch (External.Kind) {
+ case Config::ExternalIndexSpec::Server:
+ log("Associating {0} with remote index at {1}.", External.MountPoint,
+ External.Location);
+ return remote::getClient(External.Location, External.MountPoint);
+ case Config::ExternalIndexSpec::File:
+ log("Associating {0} with monolithic index at {1}.", External.MountPoint,
+ External.Location);
+ auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
+ Tasks.runAsync("Load-index:" + External.Location,
+ [File = External.Location, PlaceHolder = NewIndex.get()] {
+ if (auto Idx = loadIndex(File, /*UseDex=*/true))
+ PlaceHolder->reset(std::move(Idx));
+ });
+ return std::move(NewIndex);
+ }
+ llvm_unreachable("Invalid ExternalIndexKind.");
+}
+} // namespace
+
+std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory Gen) {
+ return std::make_unique<ProjectAwareIndex>(Gen ? std::move(Gen)
+ : defaultFactory);
+}
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/index/ProjectAware.h b/clang-tools-extra/clangd/index/ProjectAware.h
new file mode 100644
index 000000000000..2d7c6ba3ad88
--- /dev/null
+++ b/clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,34 @@
+//===--- ProjectAware.h ------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include <functional>
+#include <memory>
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification. Functor
+/// should perform any high latency operation in a separate thread through
+/// AsyncTaskRunner.
+using IndexFactory = std::function<std::unique_ptr<SymbolIndex>(
+ const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+/// The default implementation loads the index asynchronously on the
+/// AsyncTaskRunner. The index will appear empty until loaded.
+std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory = nullptr);
+} // namespace clangd
+} // namespace clang
+
+#endif
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index e570072783f1..5d87fff5c8af 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -74,6 +74,7 @@ add_unittest(ClangdUnitTests ClangdTests
PathMappingTests.cpp
PreambleTests.cpp
PrintASTTests.cpp
+ ProjectAwareIndexTests.cpp
QualityTests.cpp
RenameTests.cpp
RIFFTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp b/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
new file mode 100644
index 000000000000..8adac296ee60
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,86 @@
+//===-- ProjectAwareIndexTests.cpp -------------------*- C++ -*-----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <memory>
+#include <utility>
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr<SymbolIndex> createIndex() {
+ std::vector<Symbol> Symbols = {symbol("1")};
+ return std::make_unique<MemIndex>(std::move(Symbols), RefSlab(),
+ RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+ IndexFactory Gen = [](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+ return createIndex();
+ };
+
+ auto Idx = createProjectAwareIndex(std::move(Gen));
+ FuzzyFindRequest Req;
+ Req.Query = "1";
+ Req.AnyScope = true;
+
+ EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+ Config C;
+ C.Index.External.emplace();
+ C.Index.External->Location = "test";
+ WithContextValue With(Config::Key, std::move(C));
+ EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+ return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+ unsigned InvocationCount = 0;
+ IndexFactory Gen = [&](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+ ++InvocationCount;
+ return createIndex();
+ };
+
+ auto Idx = createProjectAwareIndex(std::move(Gen));
+ // No invocation at start.
+ EXPECT_EQ(InvocationCount, 0U);
+ FuzzyFindRequest Req;
+ Req.Query = "1";
+ Req.AnyScope = true;
+
+ // Cannot invoke without proper config.
+ match(*Idx, Req);
+ EXPECT_EQ(InvocationCount, 0U);
+
+ Config C;
+ C.Index.External.emplace();
+ C.Index.External->Location = "test";
+ WithContextValue With(Config::Key, std::move(C));
+ match(*Idx, Req);
+ // Now it should be created.
+ EXPECT_EQ(InvocationCount, 1U);
+ match(*Idx, Req);
+ // It is cached afterwards.
+ EXPECT_EQ(InvocationCount, 1U);
+ return;
+}
+} // namespace clangd
+} // namespace clang
More information about the cfe-commits
mailing list