[clang-tools-extra] r295180 - [clangd] Wire up ASTUnit and publish diagnostics with it.

Benjamin Kramer via cfe-commits cfe-commits at lists.llvm.org
Wed Feb 15 07:04:21 PST 2017


Author: d0k
Date: Wed Feb 15 09:04:20 2017
New Revision: 295180

URL: http://llvm.org/viewvc/llvm-project?rev=295180&view=rev
Log:
[clangd] Wire up ASTUnit and publish diagnostics with it.

Summary:
This requires an accessible compilation database. The parsing is done
asynchronously on a separate thread.

Reviewers: klimek, krasimir

Subscribers: cfe-commits, mgorny

Differential Revision: https://reviews.llvm.org/D29886

Added:
    clang-tools-extra/trunk/clangd/ASTManager.cpp
    clang-tools-extra/trunk/clangd/ASTManager.h
    clang-tools-extra/trunk/test/clangd/diagnostics.test
Modified:
    clang-tools-extra/trunk/clangd/CMakeLists.txt
    clang-tools-extra/trunk/clangd/ClangDMain.cpp
    clang-tools-extra/trunk/clangd/DocumentStore.h

Added: clang-tools-extra/trunk/clangd/ASTManager.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ASTManager.cpp?rev=295180&view=auto
==============================================================================
--- clang-tools-extra/trunk/clangd/ASTManager.cpp (added)
+++ clang-tools-extra/trunk/clangd/ASTManager.cpp Wed Feb 15 09:04:20 2017
@@ -0,0 +1,201 @@
+//===--- ASTManager.cpp - Clang AST manager -------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTManager.h"
+#include "JSONRPCDispatcher.h"
+#include "Protocol.h"
+#include "clang/Frontend/ASTUnit.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
+#include <mutex>
+#include <thread>
+using namespace clang;
+using namespace clangd;
+
+/// Retrieve a copy of the contents of every file in the store, for feeding into
+/// ASTUnit.
+static std::vector<ASTUnit::RemappedFile>
+getRemappedFiles(const DocumentStore &Docs) {
+  // FIXME: Use VFS instead. This would allow us to get rid of the chdir below.
+  std::vector<ASTUnit::RemappedFile> RemappedFiles;
+  for (const auto &P : Docs.getAllDocuments()) {
+    StringRef FileName = P.first;
+    FileName.consume_front("file://");
+    RemappedFiles.push_back(ASTUnit::RemappedFile(
+        FileName,
+        llvm::MemoryBuffer::getMemBufferCopy(P.second, FileName).release()));
+  }
+  return RemappedFiles;
+}
+
+/// Convert from clang diagnostic level to LSP severity.
+static int getSeverity(DiagnosticsEngine::Level L) {
+  switch (L) {
+  case DiagnosticsEngine::Remark:
+    return 4;
+  case DiagnosticsEngine::Note:
+    return 3;
+  case DiagnosticsEngine::Warning:
+    return 2;
+  case DiagnosticsEngine::Fatal:
+  case DiagnosticsEngine::Error:
+    return 1;
+  case DiagnosticsEngine::Ignored:
+    return 0;
+  }
+}
+
+ASTManager::ASTManager(JSONOutput &Output, DocumentStore &Store)
+    : Output(Output), Store(Store),
+      PCHs(std::make_shared<PCHContainerOperations>()),
+      ClangWorker([this]() { runWorker(); }) {}
+
+void ASTManager::runWorker() {
+  while (true) {
+    std::string File;
+
+    {
+      std::unique_lock<std::mutex> Lock(RequestLock);
+      // Check if there's another request pending. We keep parsing until
+      // our one-element queue is empty.
+      ClangRequestCV.wait(Lock, [this] {
+        return !RequestQueue.empty() || Done;
+      });
+
+      if (RequestQueue.empty() && Done)
+        return;
+
+      File = std::move(RequestQueue.back());
+      RequestQueue.pop_back();
+    } // unlock.
+
+    auto &Unit = ASTs[File]; // Only one thread can access this at a time.
+
+    if (!Unit) {
+      Unit = createASTUnitForFile(File, this->Store);
+    } else {
+      // Do a reparse if this wasn't the first parse.
+      // FIXME: This might have the wrong working directory if it changed in the
+      // meantime.
+      Unit->Reparse(PCHs, getRemappedFiles(this->Store));
+    }
+
+    if (!Unit)
+      continue;
+
+    // Send the diagnotics to the editor.
+    // FIXME: If the diagnostic comes from a different file, do we want to
+    // show them all? Right now we drop everything not coming from the
+    // main file.
+    // FIXME: Send FixIts to the editor.
+    std::string Diagnostics;
+    for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(),
+                                       DEnd = Unit->stored_diag_end();
+         D != DEnd; ++D) {
+      if (!D->getLocation().isValid() ||
+          !D->getLocation().getManager().isInMainFile(D->getLocation()))
+        continue;
+      Position P;
+      P.line = D->getLocation().getSpellingLineNumber() - 1;
+      P.character = D->getLocation().getSpellingColumnNumber();
+      Range R = {P, P};
+      Diagnostics +=
+          R"({"range":)" + Range::unparse(R) +
+          R"(,"severity":)" + std::to_string(getSeverity(D->getLevel())) +
+          R"(,"message":")" + llvm::yaml::escape(D->getMessage()) +
+          R"("},)";
+    }
+
+    if (!Diagnostics.empty())
+      Diagnostics.pop_back(); // Drop trailing comma.
+    Output.writeMessage(
+        R"({"jsonrpc":"2.0","method":"textDocument/publishDiagnostics","params":{"uri":")" +
+        File + R"(","diagnostics":[)" + Diagnostics + R"(]}})");
+  }
+}
+
+ASTManager::~ASTManager() {
+  {
+    std::lock_guard<std::mutex> Guard(RequestLock);
+    // Wake up the clang worker thread, then exit.
+    Done = true;
+    ClangRequestCV.notify_one();
+  }
+  ClangWorker.join();
+}
+
+void ASTManager::onDocumentAdd(StringRef Uri) {
+  std::lock_guard<std::mutex> Guard(RequestLock);
+  // Currently we discard all pending requests and just enqueue the latest one.
+  RequestQueue.clear();
+  RequestQueue.push_back(Uri);
+  ClangRequestCV.notify_one();
+}
+
+tooling::CompilationDatabase *
+ASTManager::getOrCreateCompilationDatabaseForFile(StringRef Uri) {
+  auto &I = CompilationDatabases[Uri];
+  if (I)
+    return I.get();
+
+  Uri.consume_front("file://");
+
+  std::string Error;
+  I = tooling::CompilationDatabase::autoDetectFromSource(Uri, Error);
+  Output.logs() << "Failed to load compilation database: " << Error << '\n';
+  return I.get();
+}
+
+std::unique_ptr<clang::ASTUnit>
+ASTManager::createASTUnitForFile(StringRef Uri, const DocumentStore &Docs) {
+  tooling::CompilationDatabase *CDB =
+      getOrCreateCompilationDatabaseForFile(Uri);
+
+  Uri.consume_front("file://");
+  std::vector<tooling::CompileCommand> Commands;
+
+  if (CDB) {
+    Commands = CDB->getCompileCommands(Uri);
+    // chdir. This is thread hostile.
+    if (!Commands.empty())
+      llvm::sys::fs::set_current_path(Commands.front().Directory);
+  }
+  if (Commands.empty()) {
+    // Add a fake command line if we know nothing.
+    Commands.push_back(tooling::CompileCommand(
+        llvm::sys::path::parent_path(Uri), llvm::sys::path::filename(Uri),
+        {"clang", "-fsyntax-only", Uri.str()}, ""));
+  }
+
+  // Inject the resource dir.
+  // FIXME: Don't overwrite it if it's already there.
+  static int Dummy; // Just an address in this process.
+  std::string ResourceDir =
+      CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy);
+  Commands.front().CommandLine.push_back("-resource-dir=" + ResourceDir);
+
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+      CompilerInstance::createDiagnostics(new DiagnosticOptions);
+
+  std::vector<const char *> ArgStrs;
+  for (const auto &S : Commands.front().CommandLine)
+    ArgStrs.push_back(S.c_str());
+
+  return std::unique_ptr<clang::ASTUnit>(ASTUnit::LoadFromCommandLine(
+      &*ArgStrs.begin(), &*ArgStrs.end(), PCHs, Diags, ResourceDir,
+      /*OnlyLocalDecls=*/false, /*CaptureDiagnostics=*/true,
+      getRemappedFiles(Docs),
+      /*RemappedFilesKeepOriginalName=*/true,
+      /*PrecompilePreambleAfterNParses=*/1, /*TUKind=*/TU_Complete,
+      /*CacheCodeCompletionResults=*/true,
+      /*IncludeBriefCommentsInCodeCompletion=*/true,
+      /*AllowPCHWithCompilerErrors=*/true));
+}

Added: clang-tools-extra/trunk/clangd/ASTManager.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ASTManager.h?rev=295180&view=auto
==============================================================================
--- clang-tools-extra/trunk/clangd/ASTManager.h (added)
+++ clang-tools-extra/trunk/clangd/ASTManager.h Wed Feb 15 09:04:20 2017
@@ -0,0 +1,76 @@
+//===--- ASTManager.h - Clang AST manager -----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTMANAGER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTMANAGER_H
+
+#include "DocumentStore.h"
+#include "JSONRPCDispatcher.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include <condition_variable>
+#include <deque>
+#include <thread>
+
+namespace clang {
+class ASTUnit;
+class DiagnosticsEngine;
+class PCHContainerOperations;
+namespace tooling {
+class CompilationDatabase;
+} // namespace tooling
+
+namespace clangd {
+
+class ASTManager : public DocumentStoreListener {
+public:
+  ASTManager(JSONOutput &Output, DocumentStore &Store);
+  ~ASTManager() override;
+
+  void onDocumentAdd(StringRef Uri) override;
+  // FIXME: Implement onDocumentRemove
+  // FIXME: Implement codeComplete
+
+private:
+  JSONOutput &Output;
+  DocumentStore &Store;
+
+  /// Loads a compilation database for URI. May return nullptr if it fails. The
+  /// database is cached for subsequent accesses.
+  clang::tooling::CompilationDatabase *
+  getOrCreateCompilationDatabaseForFile(StringRef Uri);
+  // Craetes a new ASTUnit for the document at Uri.
+  // FIXME: This calls chdir internally, which is thread unsafe.
+  std::unique_ptr<clang::ASTUnit>
+  createASTUnitForFile(StringRef Uri, const DocumentStore &Docs);
+
+  void runWorker();
+
+  /// Clang objects.
+  llvm::StringMap<std::unique_ptr<clang::ASTUnit>> ASTs;
+  llvm::StringMap<std::unique_ptr<clang::tooling::CompilationDatabase>>
+      CompilationDatabases;
+  std::shared_ptr<clang::PCHContainerOperations> PCHs;
+
+  /// We run parsing on a separate thread. This thread looks into PendingRequest
+  /// as a 'one element work queue' as long as RequestIsPending is true.
+  std::thread ClangWorker;
+  /// Queue of requests.
+  std::deque<std::string> RequestQueue;
+  /// Setting Done to true will make the worker thread terminate.
+  bool Done = false;
+  /// Condition variable to wake up the worker thread.
+  std::condition_variable ClangRequestCV;
+  /// Lock for accesses to RequestQueue and Done.
+  std::mutex RequestLock;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=295180&r1=295179&r2=295180&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Feb 15 09:04:20 2017
@@ -1,4 +1,5 @@
 add_clang_executable(clangd
+  ASTManager.cpp
   ClangDMain.cpp
   JSONRPCDispatcher.cpp
   Protocol.cpp
@@ -8,5 +9,7 @@ add_clang_executable(clangd
 target_link_libraries(clangd
   clangBasic
   clangFormat
+  clangFrontend
+  clangTooling
   LLVMSupport
   )

Modified: clang-tools-extra/trunk/clangd/ClangDMain.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangDMain.cpp?rev=295180&r1=295179&r2=295180&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangDMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangDMain.cpp Wed Feb 15 09:04:20 2017
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ASTManager.h"
 #include "DocumentStore.h"
 #include "JSONRPCDispatcher.h"
 #include "ProtocolHandlers.h"
@@ -27,6 +28,8 @@ int main(int argc, char *argv[]) {
   // Set up a document store and intialize all the method handlers for JSONRPC
   // dispatching.
   DocumentStore Store;
+  ASTManager AST(Out, Store);
+  Store.addListener(&AST);
   JSONRPCDispatcher Dispatcher(llvm::make_unique<Handler>(Out));
   Dispatcher.registerHandler("initialize",
                              llvm::make_unique<InitializeHandler>(Out));

Modified: clang-tools-extra/trunk/clangd/DocumentStore.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/DocumentStore.h?rev=295180&r1=295179&r2=295180&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/DocumentStore.h (original)
+++ clang-tools-extra/trunk/clangd/DocumentStore.h Wed Feb 15 09:04:20 2017
@@ -12,27 +12,65 @@
 
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringMap.h"
+#include <mutex>
 #include <string>
 
 namespace clang {
 namespace clangd {
+class DocumentStore;
+
+struct DocumentStoreListener {
+  virtual ~DocumentStoreListener() = default;
+  virtual void onDocumentAdd(StringRef Uri) {}
+  virtual void onDocumentRemove(StringRef Uri) {}
+};
 
 /// A container for files opened in a workspace, addressed by URI. The contents
 /// are owned by the DocumentStore.
 class DocumentStore {
 public:
   /// Add a document to the store. Overwrites existing contents.
-  void addDocument(StringRef Uri, StringRef Text) { Docs[Uri] = Text; }
+  void addDocument(StringRef Uri, StringRef Text) {
+    {
+      std::lock_guard<std::mutex> Guard(DocsMutex);
+      Docs[Uri] = Text;
+    }
+    for (const auto &Listener : Listeners)
+      Listener->onDocumentAdd(Uri);
+  }
   /// Delete a document from the store.
-  void removeDocument(StringRef Uri) { Docs.erase(Uri); }
+  void removeDocument(StringRef Uri) {
+    {
+      std::lock_guard<std::mutex> Guard(DocsMutex);
+      Docs.erase(Uri);
+    }
+    for (const auto &Listener : Listeners)
+      Listener->onDocumentRemove(Uri);
+  }
   /// Retrieve a document from the store. Empty string if it's unknown.
-  StringRef getDocument(StringRef Uri) const {
-    auto I = Docs.find(Uri);
-    return I == Docs.end() ? StringRef("") : StringRef(I->second);
+  std::string getDocument(StringRef Uri) const {
+    // FIXME: This could be a reader lock.
+    std::lock_guard<std::mutex> Guard(DocsMutex);
+    return Docs.lookup(Uri);
+  }
+
+  /// Add a listener. Does not take ownership.
+  void addListener(DocumentStoreListener *DSL) { Listeners.push_back(DSL); }
+
+  /// Get name and constents of all documents in this store.
+  std::vector<std::pair<std::string, std::string>> getAllDocuments() const {
+    std::vector<std::pair<std::string, std::string>> AllDocs;
+    std::lock_guard<std::mutex> Guard(DocsMutex);
+    for (const auto &P : Docs)
+      AllDocs.emplace_back(P.first(), P.second);
+    return AllDocs;
   }
 
 private:
   llvm::StringMap<std::string> Docs;
+  std::vector<DocumentStoreListener *> Listeners;
+
+  mutable std::mutex DocsMutex;
 };
 
 } // namespace clangd

Added: clang-tools-extra/trunk/test/clangd/diagnostics.test
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/diagnostics.test?rev=295180&view=auto
==============================================================================
--- clang-tools-extra/trunk/test/clangd/diagnostics.test (added)
+++ clang-tools-extra/trunk/test/clangd/diagnostics.test Wed Feb 15 09:04:20 2017
@@ -0,0 +1,17 @@
+# RUN: clangd < %s | FileCheck %s
+# It is absolutely vital that this file has CRLF line endings.
+#
+Content-Length: 125
+
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+#
+Content-Length: 152
+
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///foo.c","languageId":"c","version":1,"text":"void main() {}"}}}
+#
+# CHECK: {"jsonrpc":"2.0","method":"textDocument/publishDiagnostics","params":{"uri":"file:///foo.c","diagnostics":[{"range":{"start": {"line": 0, "character": 1}, "end": {"line": 0, "character": 1}},"severity":2,"message":"return type of 'main' is not 'int'"},{"range":{"start": {"line": 0, "character": 1}, "end": {"line": 0, "character": 1}},"severity":3,"message":"change return type to 'int'"}]}}
+#
+#
+Content-Length: 44
+
+{"jsonrpc":"2.0","id":5,"method":"shutdown"}




More information about the cfe-commits mailing list