[clang-tools-extra] cee80c0 - [clangd] Pull installed gRPC and introduce clangd-remote-(server|client)
Kirill Bobyrev via cfe-commits
cfe-commits at lists.llvm.org
Thu Apr 16 04:55:29 PDT 2020
Author: Kirill Bobyrev
Date: 2020-04-16T13:55:08+02:00
New Revision: cee80c0489e96c36269388b2aacd4da1c5714a66
URL: https://github.com/llvm/llvm-project/commit/cee80c0489e96c36269388b2aacd4da1c5714a66
DIFF: https://github.com/llvm/llvm-project/commit/cee80c0489e96c36269388b2aacd4da1c5714a66.diff
LOG: [clangd] Pull installed gRPC and introduce clangd-remote-(server|client)
Summary:
This patch allows using installed gRPC to build two simple tools which
currently provide the functionality of looking up the symbol by name.
remote-index-client is a simplified version of dexp which connects to
remote-index-server passes lookup requests.
I also significantly reduced the scope of this patch to prevent large changelist
and more bugs. The next steps would be:
* Extending Protocol for deep copies of Symbol and inherit RemoteIndex from
Index to unify the interfaces
* Make remote-index-server more generic and merge the remote index client with
dexp
* Modify Clangd to allow using remote index instead of the local one for all
global index requests
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D77794
Added:
clang-tools-extra/clangd/index/remote/CMakeLists.txt
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/README.md
clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
clang-tools-extra/clangd/index/remote/client/Client.cpp
clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
clang-tools-extra/clangd/index/remote/server/Server.cpp
llvm/cmake/modules/FindGRPC.cmake
Modified:
clang-tools-extra/clangd/CMakeLists.txt
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 7a9a4f7932ae..1c2cbf398b77 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -153,3 +153,12 @@ if(CLANG_INCLUDE_TESTS)
add_subdirectory(test)
add_subdirectory(unittests)
endif()
+
+# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+ include(FindGRPC)
+ add_subdirectory(index/remote)
+endif()
diff --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
new file mode 100644
index 000000000000..b946958f3c5f
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
@@ -0,0 +1,7 @@
+generate_grpc_protos(RemoteIndexProtos "Index.proto")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+add_subdirectory(client)
+add_subdirectory(server)
diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto
new file mode 100644
index 000000000000..399036ed72b7
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -0,0 +1,19 @@
+//===--- Index.proto - Remote index Protocol Buffers definition -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto3";
+
+package clang.clangd.remote;
+
+service Index {
+ rpc Lookup(LookupRequest) returns (stream LookupReply) {}
+}
+
+message LookupRequest { string id = 1; }
+
+message LookupReply { string symbol_yaml = 1; }
diff --git a/clang-tools-extra/clangd/index/remote/README.md b/clang-tools-extra/clangd/index/remote/README.md
new file mode 100644
index 000000000000..b56b2fc1011e
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/README.md
@@ -0,0 +1,59 @@
+# Clangd remote index
+
+Clangd uses a global index for project-wide code completion, navigation and
+other features. For large projects, building this can take many hours and
+keeping it loaded uses a lot of memory.
+
+To relieve that burden, we're building remote index — a global index
+served on a
diff erent machine and shared between developers. This directory
+contains code that is used as Proof of Concept for the upcoming remote index
+feature.
+
+## Building
+
+This feature uses gRPC and Protobuf libraries, so you will need to install them.
+There are two ways of doing that.
+
+However you install dependencies, to enable this feature and build remote index
+tools you will need to set this CMake flag — `-DCLANGD_ENABLE_REMOTE=On`.
+
+### System-installed libraries
+
+On Debian-like systems gRPC and Protobuf can be installed from apt:
+
+```bash
+apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc
+```
+
+### Building from sources
+
+Another way of installing gRPC and Protobuf is building from sources using
+CMake. The easiest way of doing that would be to choose a directory where you
+want to install so that the installation files are not copied to system root and
+you can uninstall gRPC or use
diff erent versions of the library.
+
+```bash
+# Get source code.
+$ git clone -b v1.28.1 https://github.com/grpc/grpc
+$ cd grpc
+$ git submodule update --init
+# Choose directory where you want gRPC installation to live.
+$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed
+# Build and install gRPC to ${GRPC_INSTALL_PATH}
+$ mkdir build; cd build
+$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release ..
+$ make install
+```
+
+This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into
+more detail on how to build gRPC from sources.
+
+By default, CMake will look for system-installed libraries when building remote
+index tools so you will have to adjust LLVM's CMake invocation. The following
+flag will inform build system that you chose this option —
+`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`.
+
+## Running
+
+The remote index isn't usable with Clangd yet, but you can try the
+proof-of-concept tools in `client/` and `server/` subdirectories.
diff --git a/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
new file mode 100644
index 000000000000..18bca1b04436
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS
+ LineEditor
+ Support
+ )
+add_clang_executable(clangd-index-client
+ Client.cpp
+ )
+target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-client
+ PRIVATE
+ clangDaemon
+ )
+target_link_libraries(clangd-index-client
+ PRIVATE
+ RemoteIndexProtos
+
+ protobuf
+ grpc++
+ )
diff --git a/clang-tools-extra/clangd/index/remote/client/Client.cpp b/clang-tools-extra/clangd/index/remote/client/Client.cpp
new file mode 100644
index 000000000000..5e888c5e0fa7
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/client/Client.cpp
@@ -0,0 +1,91 @@
+//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SourceCode.h"
+#include "index/Serialization.h"
+#include "index/dex/Dex.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::cl::opt<std::string>
+ ServerAddress("server-address",
+ llvm::cl::desc("Address of remote index server to use."),
+ llvm::cl::init("0.0.0.0:50051"));
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via clangd-indexer with the help
+of remote index server. The client will connect to remote index server and pass
+it lookup queries.
+)";
+
+class RemoteIndexClient {
+public:
+ RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel)
+ : Stub(remote::Index::NewStub(Channel)) {}
+
+ void lookup(llvm::StringRef ID) {
+ llvm::outs() << "Lookup of symbol with ID " << ID << '\n';
+ remote::LookupRequest Proto;
+ Proto.set_id(ID.str());
+
+ grpc::ClientContext Context;
+ remote::LookupReply Reply;
+ std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader(
+ Stub->Lookup(&Context, Proto));
+ while (Reader->Read(&Reply)) {
+ llvm::outs() << Reply.symbol_yaml();
+ }
+ grpc::Status Status = Reader->Finish();
+ if (Status.ok()) {
+ llvm::outs() << "lookupRequest rpc succeeded.\n";
+ } else {
+ llvm::outs() << "lookupRequest rpc failed.\n";
+ }
+ }
+
+private:
+ std::unique_ptr<remote::Index::Stub> Stub;
+};
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char *argv[]) {
+ using namespace clang::clangd;
+
+ llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+ llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ RemoteIndexClient IndexClient(
+ grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials()));
+
+ llvm::LineEditor LE("remote-index-client");
+ while (llvm::Optional<std::string> Request = LE.readLine())
+ IndexClient.lookup(std::move(*Request));
+}
diff --git a/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
new file mode 100644
index 000000000000..7493be1a444f
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
@@ -0,0 +1,20 @@
+set(LLVM_LINK_COMPONENTS
+ LineEditor
+ Support
+ )
+add_clang_executable(clangd-index-server
+ Server.cpp
+ )
+target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-server
+ PRIVATE
+ clangDaemon
+ )
+target_link_libraries(clangd-index-server
+ PRIVATE
+ RemoteIndexProtos
+
+ protobuf
+ grpc++
+ clangDaemon
+ )
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
new file mode 100644
index 000000000000..b7a54b79b6c3
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -0,0 +1,102 @@
+//===--- Server.cpp - gRPC-based Remote Index Server ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/Index.h"
+#include "index/Serialization.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+#include "grpcpp/health_check_service_interface.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+static const std::string Overview = R"(
+This is an experimental remote index implementation. The server opens Dex and
+awaits gRPC lookup requests from the client.
+)";
+
+llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
+ llvm::cl::Positional, llvm::cl::Required);
+
+llvm::cl::opt<std::string> ServerAddress("server-address",
+ llvm::cl::init("0.0.0.0:50051"));
+
+std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
+ return loadIndex(Index, /*UseIndex=*/true);
+}
+
+class RemoteIndexServer final : public remote::Index::Service {
+public:
+ RemoteIndexServer(std::unique_ptr<SymbolIndex> Index)
+ : Index(std::move(Index)) {}
+
+private:
+ grpc::Status Lookup(grpc::ServerContext *Context,
+ const remote::LookupRequest *Request,
+ grpc::ServerWriter<remote::LookupReply> *Reply) override {
+ llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n';
+ LookupRequest Req;
+ auto SID = SymbolID::fromStr(Request->id());
+ if (!SID) {
+ llvm::outs() << llvm::toString(SID.takeError()) << "\n";
+ return grpc::Status::CANCELLED;
+ }
+ Req.IDs.insert(*SID);
+ Index->lookup(Req, [&](const Symbol &Sym) {
+ remote::LookupReply NextSymbol;
+ NextSymbol.set_symbol_yaml(toYAML(Sym));
+ Reply->Write(NextSymbol);
+ });
+ return grpc::Status::OK;
+ }
+
+ std::unique_ptr<SymbolIndex> Index;
+};
+
+void runServer(std::unique_ptr<SymbolIndex> Index,
+ const std::string &ServerAddress) {
+ RemoteIndexServer Service(std::move(Index));
+
+ grpc::EnableDefaultHealthCheckService(true);
+ grpc::ServerBuilder Builder;
+ Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
+ Builder.RegisterService(&Service);
+ std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
+ llvm::outs() << "Server listening on " << ServerAddress << '\n';
+
+ Server->Wait();
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, char *argv[]) {
+ using namespace clang::clangd;
+ llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview);
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath);
+
+ if (!Index) {
+ llvm::outs() << "Failed to open the index.\n";
+ return -1;
+ }
+
+ runServer(std::move(Index), ServerAddress);
+}
diff --git a/llvm/cmake/modules/FindGRPC.cmake b/llvm/cmake/modules/FindGRPC.cmake
new file mode 100644
index 000000000000..b70356696298
--- /dev/null
+++ b/llvm/cmake/modules/FindGRPC.cmake
@@ -0,0 +1,50 @@
+# This setup requires gRPC to be built from sources using CMake and installed to
+# ${GRPC_INSTALL_PATH} via -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH}.
+if (GRPC_INSTALL_PATH)
+ set(protobuf_MODULE_COMPATIBLE TRUE)
+ find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using protobuf ${protobuf_VERSION}")
+ find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+ message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+ include_directories(${Protobuf_INCLUDE_DIRS})
+
+ # gRPC CMake CONFIG gives the libraries slightly odd names, make them match
+ # the conventional system-installed names.
+ set_target_properties(protobuf::libprotobuf PROPERTIES IMPORTED_GLOBAL TRUE)
+ add_library(protobuf ALIAS protobuf::libprotobuf)
+ set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_GLOBAL TRUE)
+ add_library(grpc++ ALIAS gRPC::grpc++)
+
+ set(GRPC_CPP_PLUGIN $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+ set(PROTOC ${Protobuf_PROTOC_EXECUTABLE})
+else()
+ find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin)
+ find_program(PROTOC protoc)
+endif()
+
+# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}.
+# Libraries that use these headers should adjust the include path.
+# FIXME(kirillbobyrev): Allow optional generation of gRPC code and give callers
+# control over it via additional parameters.
+function(generate_grpc_protos LibraryName ProtoFile)
+ get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE)
+ get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH)
+
+ set(GeneratedProtoSource "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc")
+ set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h")
+ set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc")
+ set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h")
+ add_custom_command(
+ OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}"
+ COMMAND ${PROTOC}
+ ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}"
+ --cpp_out="${CMAKE_CURRENT_BINARY_DIR}"
+ --proto_path="${ProtoSourcePath}"
+ --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}"
+ "${ProtoSourceAbsolutePath}"
+ DEPENDS "${ProtoSourceAbsolutePath}")
+
+ add_library(${LibraryName} ${GeneratedProtoSource} ${GeneratedGRPCSource})
+ target_link_libraries(${LibraryName} grpc++ protobuf)
+endfunction()
More information about the cfe-commits
mailing list