[clang-tools-extra] 3a0a2a6 - [clangd] Implement hot index reloading for clangd-index-server
Kirill Bobyrev via cfe-commits
cfe-commits at lists.llvm.org
Wed Sep 16 02:12:27 PDT 2020
Author: Kirill Bobyrev
Date: 2020-09-16T11:12:14+02:00
New Revision: 3a0a2a6347f5a79ebfba2cc2b763dd02001d9baa
URL: https://github.com/llvm/llvm-project/commit/3a0a2a6347f5a79ebfba2cc2b763dd02001d9baa
DIFF: https://github.com/llvm/llvm-project/commit/3a0a2a6347f5a79ebfba2cc2b763dd02001d9baa.diff
LOG: [clangd] Implement hot index reloading for clangd-index-server
This patch adds a mechanism to load new versions of index into
clangd-index-server using SwapIndex and FileStatus information about last
modification time without downtime.
Reviewed By: kadircet
Differential Revision: https://reviews.llvm.org/D87450
Added:
Modified:
clang-tools-extra/clangd/index/remote/server/Server.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index e9838cce85e3..d8cf54249662 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -12,15 +12,25 @@
#include "index/Symbol.h"
#include "index/remote/marshalling/Marshalling.h"
#include "support/Logger.h"
+#include "support/Shutdown.h"
+#include "support/ThreadsafeFS.h"
#include "support/Trace.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/None.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Chrono.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <chrono>
#include <grpc++/grpc++.h>
#include <grpc++/health_check_service_interface.h>
+#include <memory>
+#include <thread>
#include "Index.grpc.pb.h"
@@ -63,15 +73,10 @@ llvm::cl::opt<std::string> ServerAddress(
"server-address", llvm::cl::init("0.0.0.0:50051"),
llvm::cl::desc("Address of the invoked server. Defaults to 0.0.0.0:50051"));
-std::unique_ptr<clangd::SymbolIndex> openIndex(llvm::StringRef Index) {
- return loadIndex(Index, /*UseIndex=*/true);
-}
-
class RemoteIndexServer final : public SymbolIndex::Service {
public:
- RemoteIndexServer(std::unique_ptr<clangd::SymbolIndex> Index,
- llvm::StringRef IndexRoot)
- : Index(std::move(Index)) {
+ RemoteIndexServer(clangd::SymbolIndex &Index, llvm::StringRef IndexRoot)
+ : Index(Index) {
llvm::SmallString<256> NativePath = IndexRoot;
llvm::sys::path::native(NativePath);
ProtobufMarshaller = std::unique_ptr<Marshaller>(new Marshaller(
@@ -91,7 +96,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
}
unsigned Sent = 0;
unsigned FailedToSend = 0;
- Index->lookup(*Req, [&](const clangd::Symbol &Item) {
+ Index.lookup(*Req, [&](const clangd::Symbol &Item) {
auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
if (!SerializedItem) {
elog("Unable to convert Symbol to protobuf: {0}",
@@ -124,7 +129,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
}
unsigned Sent = 0;
unsigned FailedToSend = 0;
- bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Item) {
+ bool HasMore = Index.fuzzyFind(*Req, [&](const clangd::Symbol &Item) {
auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
if (!SerializedItem) {
elog("Unable to convert Symbol to protobuf: {0}",
@@ -155,7 +160,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
}
unsigned Sent = 0;
unsigned FailedToSend = 0;
- bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Item) {
+ bool HasMore = Index.refs(*Req, [&](const clangd::Ref &Item) {
auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
if (!SerializedItem) {
elog("Unable to convert Ref to protobuf: {0}",
@@ -188,7 +193,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
}
unsigned Sent = 0;
unsigned FailedToSend = 0;
- Index->relations(
+ Index.relations(
*Req, [&](const SymbolID &Subject, const clangd::Symbol &Object) {
auto SerializedItem = ProtobufMarshaller->toProtobuf(Subject, Object);
if (!SerializedItem) {
@@ -210,22 +215,56 @@ class RemoteIndexServer final : public SymbolIndex::Service {
return grpc::Status::OK;
}
- std::unique_ptr<clangd::SymbolIndex> Index;
std::unique_ptr<Marshaller> ProtobufMarshaller;
+ clangd::SymbolIndex &Index;
};
-void runServer(std::unique_ptr<clangd::SymbolIndex> Index,
- const std::string &ServerAddress) {
- RemoteIndexServer Service(std::move(Index), IndexRoot);
+// Detect changes in \p IndexPath file and load new versions of the index
+// whenever they become available.
+void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath,
+ llvm::vfs::Status &LastStatus,
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &FS) {
+ auto Status = FS->status(IndexPath);
+ // Requested file is same as loaded index: no reload is needed.
+ if (!Status || (Status->getLastModificationTime() ==
+ LastStatus.getLastModificationTime() &&
+ Status->getSize() == LastStatus.getSize()))
+ return;
+ vlog("Found
diff erent index version: existing index was modified at {0}, new "
+ "index was modified at {1}. Attempting to reload.",
+ LastStatus.getLastModificationTime(), Status->getLastModificationTime());
+ LastStatus = *Status;
+ std::unique_ptr<clang::clangd::SymbolIndex> NewIndex = loadIndex(IndexPath);
+ if (!NewIndex) {
+ elog("Failed to load new index. Old index will be served.");
+ return;
+ }
+ Index.reset(std::move(NewIndex));
+ log("New index version loaded. Last modification time: {0}, size: {1} bytes.",
+ Status->getLastModificationTime(), Status->getSize());
+}
+
+void runServerAndWait(clangd::SymbolIndex &Index, llvm::StringRef ServerAddress,
+ llvm::StringRef IndexPath) {
+ RemoteIndexServer Service(Index, IndexRoot);
grpc::EnableDefaultHealthCheckService(true);
grpc::ServerBuilder Builder;
- Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
+ Builder.AddListeningPort(ServerAddress.str(),
+ grpc::InsecureServerCredentials());
Builder.RegisterService(&Service);
std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
log("Server listening on {0}", ServerAddress);
+ std::thread ServerShutdownWatcher([&]() {
+ static constexpr auto WatcherFrequency = std::chrono::seconds(5);
+ while (!clang::clangd::shutdownRequested())
+ std::this_thread::sleep_for(WatcherFrequency);
+ Server->Shutdown();
+ });
+
Server->Wait();
+ ServerShutdownWatcher.join();
}
} // namespace
@@ -239,6 +278,7 @@ int main(int argc, char *argv[]) {
using namespace clang::clangd::remote;
llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+ llvm::sys::SetInterruptFunction(&clang::clangd::requestShutdown);
if (!llvm::sys::path::is_absolute(IndexRoot)) {
llvm::errs() << "Index root should be an absolute path.\n";
@@ -273,12 +313,32 @@ int main(int argc, char *argv[]) {
if (Tracer)
TracingSession.emplace(*Tracer);
- std::unique_ptr<clang::clangd::SymbolIndex> Index = openIndex(IndexPath);
+ clang::clangd::RealThreadsafeFS TFS;
+ auto FS = TFS.view(llvm::None);
+ auto Status = FS->status(IndexPath);
+ if (!Status) {
+ elog("{0} does not exist.", IndexPath);
+ return Status.getError().value();
+ }
+
+ auto Index = std::make_unique<clang::clangd::SwapIndex>(
+ clang::clangd::loadIndex(IndexPath));
if (!Index) {
llvm::errs() << "Failed to open the index.\n";
return -1;
}
- runServer(std::move(Index), ServerAddress);
+ std::thread HotReloadThread([&Index, &Status, &FS]() {
+ llvm::vfs::Status LastStatus = *Status;
+ static constexpr auto RefreshFrequency = std::chrono::seconds(90);
+ while (!clang::clangd::shutdownRequested()) {
+ hotReload(*Index, llvm::StringRef(IndexPath), LastStatus, FS);
+ std::this_thread::sleep_for(RefreshFrequency);
+ }
+ });
+
+ runServerAndWait(*Index, ServerAddress, IndexPath);
+
+ HotReloadThread.join();
}
More information about the cfe-commits
mailing list