[llvm-branch-commits] [clang-tools-extra] 067ffbf - [clangd] Introduce ProjectAwareIndex

Kadir Cetinkaya via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 22 12:17:49 PST 2020


Author: Kadir Cetinkaya
Date: 2020-11-22T20:59:37+01:00
New Revision: 067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76

URL: https://github.com/llvm/llvm-project/commit/067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76
DIFF: https://github.com/llvm/llvm-project/commit/067ffbfe60180aa7b1fdd87b2b6e8ccc67a43a76.diff

LOG: [clangd] Introduce ProjectAwareIndex

An index implementation that can dispatch to a variety of indexes
depending on the file path. Enables clangd to work with multiple indexes in the
same instance, configured via config files.

Depends on D90749, D90746

Differential Revision: https://reviews.llvm.org/D90750

Added: 
    clang-tools-extra/clangd/index/ProjectAware.cpp
    clang-tools-extra/clangd/index/ProjectAware.h
    clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Modified: 
    clang-tools-extra/clangd/CMakeLists.txt
    clang-tools-extra/clangd/Config.h
    clang-tools-extra/clangd/unittests/CMakeLists.txt

Removed: 
    


################################################################################
diff  --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 3fd110e9e135..d02a5cf3f2ec 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -97,6 +97,7 @@ add_clang_library(clangDaemon
   index/IndexAction.cpp
   index/MemIndex.cpp
   index/Merge.cpp
+  index/ProjectAware.cpp
   index/Ref.cpp
   index/Relation.cpp
   index/Serialization.cpp
@@ -150,6 +151,7 @@ target_link_libraries(clangDaemon
   clangTidy
   ${ALL_CLANG_TIDY_CHECKS}
 
+  clangdRemoteIndex
   clangdSupport
   )
 

diff  --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index 318220364e6e..79e94ef6fe37 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -97,4 +97,24 @@ struct Config {
 } // namespace clangd
 } // namespace clang
 
+namespace llvm {
+template <> struct DenseMapInfo<clang::clangd::Config::ExternalIndexSpec> {
+  using ExternalIndexSpec = clang::clangd::Config::ExternalIndexSpec;
+  static inline ExternalIndexSpec getEmptyKey() {
+    return {ExternalIndexSpec::File, "", ""};
+  }
+  static inline ExternalIndexSpec getTombstoneKey() {
+    return {ExternalIndexSpec::File, "TOMB", "STONE"};
+  }
+  static unsigned getHashValue(const ExternalIndexSpec &Val) {
+    return llvm::hash_combine(Val.Kind, Val.Location, Val.MountPoint);
+  }
+  static bool isEqual(const ExternalIndexSpec &LHS,
+                      const ExternalIndexSpec &RHS) {
+    return std::tie(LHS.Kind, LHS.Location, LHS.MountPoint) ==
+           std::tie(RHS.Kind, RHS.Location, RHS.MountPoint);
+  }
+};
+} // namespace llvm
+
 #endif

diff  --git a/clang-tools-extra/clangd/index/ProjectAware.cpp b/clang-tools-extra/clangd/index/ProjectAware.cpp
new file mode 100644
index 000000000000..58685fd14bf5
--- /dev/null
+++ b/clang-tools-extra/clangd/index/ProjectAware.cpp
@@ -0,0 +1,156 @@
+//===--- ProjectAware.h ------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProjectAware.h"
+#include "Config.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/Merge.h"
+#include "index/Ref.h"
+#include "index/Serialization.h"
+#include "index/Symbol.h"
+#include "index/SymbolID.h"
+#include "index/remote/Client.h"
+#include "support/Logger.h"
+#include "support/Threading.h"
+#include "support/Trace.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <map>
+#include <memory>
+#include <mutex>
+#include <tuple>
+
+namespace clang {
+namespace clangd {
+namespace {
+class ProjectAwareIndex : public SymbolIndex {
+public:
+  size_t estimateMemoryUsage() const override;
+
+  /// Only queries the associated index with the current context.
+  void lookup(const LookupRequest &Req,
+              llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  bool refs(const RefsRequest &Req,
+            llvm::function_ref<void(const Ref &)> Callback) const override;
+
+  /// Queries only the associates index when Req.RestrictForCodeCompletion is
+  /// set, otherwise queries all.
+  bool
+  fuzzyFind(const FuzzyFindRequest &Req,
+            llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  void relations(const RelationsRequest &Req,
+                 llvm::function_ref<void(const SymbolID &, const Symbol &)>
+                     Callback) const override;
+
+  ProjectAwareIndex(IndexFactory Gen) : Gen(std::move(Gen)) {}
+
+private:
+  // Returns the index associated with current context, if any.
+  SymbolIndex *getIndex() const;
+
+  // Storage for all the external indexes.
+  mutable std::mutex Mu;
+  mutable llvm::DenseMap<Config::ExternalIndexSpec,
+                         std::unique_ptr<SymbolIndex>>
+      IndexForSpec;
+  mutable AsyncTaskRunner Tasks;
+
+  const IndexFactory Gen;
+};
+
+size_t ProjectAwareIndex::estimateMemoryUsage() const {
+  size_t Total = 0;
+  std::lock_guard<std::mutex> Lock(Mu);
+  for (auto &Entry : IndexForSpec)
+    Total += Entry.second->estimateMemoryUsage();
+  return Total;
+}
+
+void ProjectAwareIndex::lookup(
+    const LookupRequest &Req,
+    llvm::function_ref<void(const Symbol &)> Callback) const {
+  trace::Span Tracer("ProjectAwareIndex::lookup");
+  if (auto *Idx = getIndex())
+    Idx->lookup(Req, Callback);
+}
+
+bool ProjectAwareIndex::refs(
+    const RefsRequest &Req,
+    llvm::function_ref<void(const Ref &)> Callback) const {
+  trace::Span Tracer("ProjectAwareIndex::refs");
+  if (auto *Idx = getIndex())
+    return Idx->refs(Req, Callback);
+  return false;
+}
+
+bool ProjectAwareIndex::fuzzyFind(
+    const FuzzyFindRequest &Req,
+    llvm::function_ref<void(const Symbol &)> Callback) const {
+  trace::Span Tracer("ProjectAwareIndex::fuzzyFind");
+  if (auto *Idx = getIndex())
+    return Idx->fuzzyFind(Req, Callback);
+  return false;
+}
+
+void ProjectAwareIndex::relations(
+    const RelationsRequest &Req,
+    llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
+  trace::Span Tracer("ProjectAwareIndex::relations");
+  if (auto *Idx = getIndex())
+    return Idx->relations(Req, Callback);
+}
+
+SymbolIndex *ProjectAwareIndex::getIndex() const {
+  const auto &C = Config::current();
+  if (!C.Index.External)
+    return nullptr;
+  const auto &External = *C.Index.External;
+  std::lock_guard<std::mutex> Lock(Mu);
+  auto Entry = IndexForSpec.try_emplace(External, nullptr);
+  if (Entry.second)
+    Entry.first->getSecond() = Gen(External, Tasks);
+  return Entry.first->second.get();
+}
+
+std::unique_ptr<SymbolIndex>
+defaultFactory(const Config::ExternalIndexSpec &External,
+               AsyncTaskRunner &Tasks) {
+  switch (External.Kind) {
+  case Config::ExternalIndexSpec::Server:
+    log("Associating {0} with remote index at {1}.", External.MountPoint,
+        External.Location);
+    return remote::getClient(External.Location, External.MountPoint);
+  case Config::ExternalIndexSpec::File:
+    log("Associating {0} with monolithic index at {1}.", External.MountPoint,
+        External.Location);
+    auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
+    Tasks.runAsync("Load-index:" + External.Location,
+                   [File = External.Location, PlaceHolder = NewIndex.get()] {
+                     if (auto Idx = loadIndex(File, /*UseDex=*/true))
+                       PlaceHolder->reset(std::move(Idx));
+                   });
+    return std::move(NewIndex);
+  }
+  llvm_unreachable("Invalid ExternalIndexKind.");
+}
+} // namespace
+
+std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory Gen) {
+  return std::make_unique<ProjectAwareIndex>(Gen ? std::move(Gen)
+                                                 : defaultFactory);
+}
+} // namespace clangd
+} // namespace clang

diff  --git a/clang-tools-extra/clangd/index/ProjectAware.h b/clang-tools-extra/clangd/index/ProjectAware.h
new file mode 100644
index 000000000000..2d7c6ba3ad88
--- /dev/null
+++ b/clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,34 @@
+//===--- ProjectAware.h ------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include <functional>
+#include <memory>
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification. Functor
+/// should perform any high latency operation in a separate thread through
+/// AsyncTaskRunner.
+using IndexFactory = std::function<std::unique_ptr<SymbolIndex>(
+    const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+/// The default implementation loads the index asynchronously on the
+/// AsyncTaskRunner. The index will appear empty until loaded.
+std::unique_ptr<SymbolIndex> createProjectAwareIndex(IndexFactory = nullptr);
+} // namespace clangd
+} // namespace clang
+
+#endif

diff  --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index e570072783f1..5d87fff5c8af 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -74,6 +74,7 @@ add_unittest(ClangdUnitTests ClangdTests
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp

diff  --git a/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp b/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
new file mode 100644
index 000000000000..8adac296ee60
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,86 @@
+//===-- ProjectAwareIndexTests.cpp  -------------------*- C++ -*-----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <memory>
+#include <utility>
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr<SymbolIndex> createIndex() {
+  std::vector<Symbol> Symbols = {symbol("1")};
+  return std::make_unique<MemIndex>(std::move(Symbols), RefSlab(),
+                                    RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+  IndexFactory Gen = [](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+    return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+  return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+  unsigned InvocationCount = 0;
+  IndexFactory Gen = [&](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+    ++InvocationCount;
+    return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  // No invocation at start.
+  EXPECT_EQ(InvocationCount, 0U);
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  // Cannot invoke without proper config.
+  match(*Idx, Req);
+  EXPECT_EQ(InvocationCount, 0U);
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  match(*Idx, Req);
+  // Now it should be created.
+  EXPECT_EQ(InvocationCount, 1U);
+  match(*Idx, Req);
+  // It is cached afterwards.
+  EXPECT_EQ(InvocationCount, 1U);
+  return;
+}
+} // namespace clangd
+} // namespace clang


        


More information about the llvm-branch-commits mailing list