[clang-tools-extra] 71177ac - [clangd] Support new semanticTokens request from LSP 3.16.
Sam McCall via cfe-commits
cfe-commits at lists.llvm.org
Tue Mar 31 06:14:44 PDT 2020
Author: Sam McCall
Date: 2020-03-31T15:14:35+02:00
New Revision: 71177ac16801ceced4b7dcdd21b05345416f31df
URL: https://github.com/llvm/llvm-project/commit/71177ac16801ceced4b7dcdd21b05345416f31df
DIFF: https://github.com/llvm/llvm-project/commit/71177ac16801ceced4b7dcdd21b05345416f31df.diff
LOG: [clangd] Support new semanticTokens request from LSP 3.16.
Summary:
This is a simpler request/response protocol.
Reference: https://github.com/microsoft/vscode-languageserver-node/blob/master/protocol/src/protocol.semanticTokens.proposed.ts
No attempt to support incremental formatting (yet).
Reviewers: hokein
Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D76663
Added:
clang-tools-extra/clangd/test/semantic-tokens.test
Modified:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/ClangdLSPServer.h
clang-tools-extra/clangd/ClangdServer.cpp
clang-tools-extra/clangd/ClangdServer.h
clang-tools-extra/clangd/Protocol.cpp
clang-tools-extra/clangd/Protocol.h
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/test/initialize-params.test
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 9be449660eb0..310c1fec17dd 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -458,6 +458,14 @@ void ClangdLSPServer::notify(llvm::StringRef Method, llvm::json::Value Params) {
Transp.notify(Method, std::move(Params));
}
+static std::vector<llvm::StringRef> semanticTokenTypes() {
+ std::vector<llvm::StringRef> Types;
+ for (unsigned I = 0; I <= static_cast<unsigned>(HighlightingKind::LastKind);
+ ++I)
+ Types.push_back(toSemanticTokenType(static_cast<HighlightingKind>(I)));
+ return Types;
+}
+
void ClangdLSPServer::onInitialize(const InitializeParams &Params,
Callback<llvm::json::Value> Reply) {
// Determine character encoding first as it affects constructed ClangdServer.
@@ -569,6 +577,14 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
// trigger on '->' and '::'.
{"triggerCharacters", {".", ">", ":"}},
}},
+ {"semanticTokensProvider",
+ llvm::json::Object{
+ {"documentProvider", true},
+ {"rangeProvider", false},
+ {"legend",
+ llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
+ {"tokenModifiers", llvm::json::Array()}}},
+ }},
{"signatureHelpProvider",
llvm::json::Object{
{"triggerCharacters", {"(", ","}},
@@ -1220,6 +1236,20 @@ void ClangdLSPServer::onDocumentLink(
});
}
+void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params,
+ Callback<SemanticTokens> CB) {
+ Server->semanticHighlights(
+ Params.textDocument.uri.file(),
+ [CB(std::move(CB))](
+ llvm::Expected<std::vector<HighlightingToken>> Toks) mutable {
+ if (!Toks)
+ return CB(Toks.takeError());
+ SemanticTokens Result;
+ Result.data = toSemanticTokens(*Toks);
+ CB(std::move(Result));
+ });
+}
+
ClangdLSPServer::ClangdLSPServer(
class Transport &Transp, const FileSystemProvider &FSProvider,
const clangd::CodeCompleteOptions &CCOpts,
@@ -1267,6 +1297,7 @@ ClangdLSPServer::ClangdLSPServer(
MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink);
+ MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens);
// clang-format on
}
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index c4e9e5fb679c..ff67bf772b7f 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -119,6 +119,7 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
Callback<std::vector<SelectionRange>>);
void onDocumentLink(const DocumentLinkParams &,
Callback<std::vector<DocumentLink>>);
+ void onSemanticTokens(const SemanticTokensParams &, Callback<SemanticTokens>);
std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index acfaa81dbf29..36af6c98d18b 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -685,6 +685,18 @@ void ClangdServer::documentLinks(PathRef File,
TUScheduler::InvalidateOnUpdate);
}
+void ClangdServer::semanticHighlights(
+ PathRef File, Callback<std::vector<HighlightingToken>> CB) {
+ auto Action =
+ [CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
+ if (!InpAST)
+ return CB(InpAST.takeError());
+ CB(clangd::getSemanticHighlightings(InpAST->AST));
+ };
+ WorkScheduler.runWithAST("SemanticHighlights", File, std::move(Action),
+ TUScheduler::InvalidateOnUpdate);
+}
+
std::vector<std::pair<Path, std::size_t>>
ClangdServer::getUsedBytesPerFile() const {
return WorkScheduler.getUsedBytesPerFile();
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index 8d9193366d95..a0659c7c3d22 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -297,7 +297,10 @@ class ClangdServer {
/// Get all document links in a file.
void documentLinks(PathRef File, Callback<std::vector<DocumentLink>> CB);
-
+
+ void semanticHighlights(PathRef File,
+ Callback<std::vector<HighlightingToken>>);
+
/// Returns estimated memory usage for each of the currently open files.
/// The order of results is unspecified.
/// Overall memory usage of clangd may be significantly more than reported
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index 462aebc4b046..9d7c96df02c0 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -984,6 +984,29 @@ llvm::json::Value toJSON(const FileStatus &FStatus) {
};
}
+void SemanticToken::encode(std::vector<unsigned int> &Out) const {
+ Out.push_back(deltaLine);
+ Out.push_back(deltaStart);
+ Out.push_back(length);
+ Out.push_back(tokenType);
+ Out.push_back(tokenModifiers);
+}
+
+llvm::json::Value toJSON(const SemanticTokens &Tokens) {
+ std::vector<unsigned> Data;
+ for (const auto &Tok : Tokens.data)
+ Tok.encode(Data);
+ llvm::json::Object Result{{"data", std::move(Data)}};
+ if (Tokens.resultId)
+ Result["resultId"] = *Tokens.resultId;
+ return Result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) {
+ llvm::json::ObjectMapper O(Params);
+ return O && O.map("textDocument", R.textDocument);
+}
+
llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
const DocumentHighlight &V) {
O << V.range;
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index d08e546e3ffb..5d0b60d7fe9f 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -1340,7 +1340,47 @@ struct FileStatus {
std::string state;
// FIXME: add detail messages.
};
-llvm::json::Value toJSON(const FileStatus &FStatus);
+llvm::json::Value toJSON(const FileStatus &);
+
+/// Specifies a single semantic token in the document.
+/// This struct is not part of LSP, which just encodes lists of tokens as
+/// arrays of numbers directly.
+struct SemanticToken {
+ /// token line number, relative to the previous token
+ unsigned deltaLine = 0;
+ /// token start character, relative to the previous token
+ /// (relative to 0 or the previous token's start if they are on the same line)
+ unsigned deltaStart = 0;
+ /// the length of the token. A token cannot be multiline
+ unsigned length = 0;
+ /// will be looked up in `SemanticTokensLegend.tokenTypes`
+ unsigned tokenType = 0;
+ /// each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+ unsigned tokenModifiers = 0;
+
+ void encode(std::vector<unsigned> &Out) const;
+};
+
+/// A versioned set of tokens.
+struct SemanticTokens {
+ // An optional result id. If provided and clients support delta updating
+ // the client will include the result id in the next semantic token request.
+ // A server can then instead of computing all semantic tokens again simply
+ // send a delta.
+ llvm::Optional<std::string> resultId;
+
+ /// The actual tokens. For a detailed description about how the data is
+ /// structured pls see
+ /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
+ std::vector<SemanticToken> data;
+};
+llvm::json::Value toJSON(const SemanticTokens &);
+
+struct SemanticTokensParams {
+ /// The text document.
+ TextDocumentIdentifier textDocument;
+};
+bool fromJSON(const llvm::json::Value &, SemanticTokensParams &);
/// Represents a semantic highlighting information that has to be applied on a
/// specific line of the text document.
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index b69f9e8f2710..77b2cbce40d9 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -445,6 +445,83 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
}
+std::vector<SemanticToken>
+toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
+ assert(std::is_sorted(Tokens.begin(), Tokens.end()));
+ std::vector<SemanticToken> Result;
+ const HighlightingToken *Last = nullptr;
+ for (const HighlightingToken &Tok : Tokens) {
+ // FIXME: support inactive code - we need to provide the actual bounds.
+ if (Tok.Kind == HighlightingKind::InactiveCode)
+ continue;
+ Result.emplace_back();
+ SemanticToken &Out = Result.back();
+ // deltaStart/deltaLine are relative if possible.
+ if (Last) {
+ assert(Tok.R.start.line >= Last->R.start.line);
+ Out.deltaLine = Tok.R.start.line - Last->R.start.line;
+ if (Out.deltaLine == 0) {
+ assert(Tok.R.start.character >= Last->R.start.character);
+ Out.deltaStart = Tok.R.start.character - Last->R.start.character;
+ } else {
+ Out.deltaStart = Tok.R.start.character;
+ }
+ } else {
+ Out.deltaLine = Tok.R.start.line;
+ Out.deltaStart = Tok.R.start.character;
+ }
+ assert(Tok.R.end.line == Tok.R.start.line);
+ Out.length = Tok.R.end.character - Tok.R.start.character;
+ Out.tokenType = static_cast<unsigned>(Tok.Kind);
+
+ Last = &Tok;
+ }
+ return Result;
+}
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
+ switch (Kind) {
+ case HighlightingKind::Variable:
+ case HighlightingKind::LocalVariable:
+ case HighlightingKind::StaticField:
+ return "variable";
+ case HighlightingKind::Parameter:
+ return "parameter";
+ case HighlightingKind::Function:
+ return "function";
+ case HighlightingKind::Method:
+ return "member";
+ case HighlightingKind::StaticMethod:
+ // FIXME: better function/member with static modifier?
+ return "function";
+ case HighlightingKind::Field:
+ return "member";
+ case HighlightingKind::Class:
+ return "class";
+ case HighlightingKind::Enum:
+ return "enum";
+ case HighlightingKind::EnumConstant:
+ return "enumConstant"; // nonstandard
+ case HighlightingKind::Typedef:
+ return "type";
+ case HighlightingKind::DependentType:
+ return "dependent"; // nonstandard
+ case HighlightingKind::DependentName:
+ return "dependent"; // nonstandard
+ case HighlightingKind::Namespace:
+ return "namespace";
+ case HighlightingKind::TemplateParameter:
+ return "typeParameter";
+ case HighlightingKind::Concept:
+ return "concept"; // nonstandard
+ case HighlightingKind::Primitive:
+ return "type";
+ case HighlightingKind::Macro:
+ return "macro";
+ case HighlightingKind::InactiveCode:
+ return "comment";
+ }
+}
+
std::vector<TheiaSemanticHighlightingInformation>
toTheiaSemanticHighlightingInformation(
llvm::ArrayRef<LineHighlightings> Tokens) {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h
index 31a97b81d6c2..d3b9ddc23a47 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.h
+++ b/clang-tools-extra/clangd/SemanticHighlighting.h
@@ -6,8 +6,21 @@
//
//===----------------------------------------------------------------------===//
//
-// An implementation of semantic highlighting based on this proposal:
-// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd.
+// This file supports semantic highlighting: categorizing tokens in the file so
+// that the editor can color/style them
diff erently.
+//
+// This is particularly valuable for C++: its complex and context-dependent
+// grammar is a challenge for simple syntax-highlighting techniques.
+//
+// We support two protocols for providing highlights to the client:
+// - the `textDocument/semanticTokens` request from LSP 3.16
+// https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts
+// - the earlier proposed `textDocument/semanticHighlighting` notification
+// https://github.com/microsoft/vscode-languageserver-node/pull/367
+// This is referred to as "Theia" semantic highlighting in the code.
+// It was supported from clangd 9 but should be considered deprecated as of
+// clangd 11 and eventually removed.
+//
// Semantic highlightings are calculated for an AST by visiting every AST node
// and classifying nodes that are interesting to highlight (variables/function
// calls etc.).
@@ -75,6 +88,9 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R);
// main AST.
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
+std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>);
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
+
/// Converts a HighlightingKind to a corresponding TextMate scope
/// (https://manual.macromates.com/en/language_grammars).
llvm::StringRef toTextMateScope(HighlightingKind Kind);
diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test
index 2b5c02fc8ce2..9eef5fa68dc9 100644
--- a/clang-tools-extra/clangd/test/initialize-params.test
+++ b/clang-tools-extra/clangd/test/initialize-params.test
@@ -38,6 +38,16 @@
# CHECK-NEXT: "referencesProvider": true,
# CHECK-NEXT: "renameProvider": true,
# CHECK-NEXT: "selectionRangeProvider": true,
+# CHECK-NEXT: "semanticTokensProvider": {
+# CHECK-NEXT: "documentProvider": true,
+# CHECK-NEXT: "legend": {
+# CHECK-NEXT: "tokenModifiers": [],
+# CHECK-NEXT: "tokenTypes": [
+# CHECK-NEXT: "variable",
+# CHECK: ]
+# CHECK-NEXT: },
+# CHECK-NEXT: "rangeProvider": false
+# CHECK-NEXT: },
# CHECK-NEXT: "signatureHelpProvider": {
# CHECK-NEXT: "triggerCharacters": [
# CHECK-NEXT: "(",
diff --git a/clang-tools-extra/clangd/test/semantic-tokens.test b/clang-tools-extra/clangd/test/semantic-tokens.test
new file mode 100644
index 000000000000..1f6b51af84dd
--- /dev/null
+++ b/clang-tools-extra/clangd/test/semantic-tokens.test
@@ -0,0 +1,22 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}}
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
+# CHECK: "id": 1,
+# CHECK-NEXT: "jsonrpc": "2.0",
+# CHECK-NEXT: "result": {
+# CHECK-NEXT: "data": [
+# First line, char 5, variable, no modifiers.
+# CHECK-NEXT: 0,
+# CHECK-NEXT: 4,
+# CHECK-NEXT: 1,
+# CHECK-NEXT: 0,
+# CHECK-NEXT: 0
+# CHECK-NEXT: ]
+# CHECK-NEXT: }
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index 6a7f700e1f49..6c4a4590ab2a 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -720,6 +720,41 @@ TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
ASSERT_EQ(Counter.Count, 1);
}
+TEST(SemanticHighlighting, toSemanticTokens) {
+ auto CreatePosition = [](int Line, int Character) -> Position {
+ Position Pos;
+ Pos.line = Line;
+ Pos.character = Character;
+ return Pos;
+ };
+
+ std::vector<HighlightingToken> Tokens = {
+ {HighlightingKind::Variable,
+ Range{CreatePosition(1, 1), CreatePosition(1, 5)}},
+ {HighlightingKind::Function,
+ Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
+ {HighlightingKind::Variable,
+ Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
+ };
+
+ std::vector<SemanticToken> Results = toSemanticTokens(Tokens);
+ EXPECT_EQ(Tokens.size(), Results.size());
+ EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
+ EXPECT_EQ(Results[0].deltaLine, 1u);
+ EXPECT_EQ(Results[0].deltaStart, 1u);
+ EXPECT_EQ(Results[0].length, 4u);
+
+ EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
+ EXPECT_EQ(Results[1].deltaLine, 2u);
+ EXPECT_EQ(Results[1].deltaStart, 4u);
+ EXPECT_EQ(Results[1].length, 3u);
+
+ EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
+ EXPECT_EQ(Results[2].deltaLine, 0u);
+ EXPECT_EQ(Results[2].deltaStart, 4u);
+ EXPECT_EQ(Results[2].length, 4u);
+}
+
TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
auto CreatePosition = [](int Line, int Character) -> Position {
Position Pos;
More information about the cfe-commits
mailing list