[clang-tools-extra] 4dc8365 - [clangd] Remove support for pre-standard semanticHighlighting notification
Sam McCall via cfe-commits
cfe-commits at lists.llvm.org
Wed Feb 10 13:12:21 PST 2021
Author: Sam McCall
Date: 2021-02-10T22:09:03+01:00
New Revision: 4dc8365f80823d8042ba4bdc3278de7a42619c24
URL: https://github.com/llvm/llvm-project/commit/4dc8365f80823d8042ba4bdc3278de7a42619c24
DIFF: https://github.com/llvm/llvm-project/commit/4dc8365f80823d8042ba4bdc3278de7a42619c24.diff
LOG: [clangd] Remove support for pre-standard semanticHighlighting notification
This is obsoleted by the standard semanticTokens request family.
As well as the protocol details, this allows us to remove a bunch of plumbing
around pushing highlights to clients.
This should not land until the new protocol has feature parity, see D77702.
Differential Revision: https://reviews.llvm.org/D95576
Added:
Modified:
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/ClangdLSPServer.h
clang-tools-extra/clangd/ClangdServer.cpp
clang-tools-extra/clangd/ClangdServer.h
clang-tools-extra/clangd/Protocol.cpp
clang-tools-extra/clangd/Protocol.h
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/TUScheduler.cpp
clang-tools-extra/clangd/TUScheduler.h
clang-tools-extra/clangd/test/semantic-tokens.test
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
Removed:
clang-tools-extra/clangd/test/semantic-highlighting.test
################################################################################
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 006b03531836..4263edb8cd6a 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -117,18 +117,6 @@ CompletionItemKindBitset defaultCompletionItemKinds() {
return Defaults;
}
-// Build a lookup table (HighlightingKind => {TextMate Scopes}), which is sent
-// to the LSP client.
-std::vector<std::vector<std::string>> buildHighlightScopeLookupTable() {
- std::vector<std::vector<std::string>> LookupTable;
- // HighlightingKind is using as the index.
- for (int KindValue = 0; KindValue <= (int)HighlightingKind::LastKind;
- ++KindValue)
- LookupTable.push_back(
- {std::string(toTextMateScope((HighlightingKind)(KindValue)))});
- return LookupTable;
-}
-
// Makes sure edits in \p FE are applicable to latest file contents reported by
// editor. If not generates an error message containing information about files
// that needs to be saved.
@@ -511,18 +499,10 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
}
}
- Opts.TheiaSemanticHighlighting =
- Params.capabilities.TheiaSemanticHighlighting;
if (Params.capabilities.TheiaSemanticHighlighting &&
- Params.capabilities.SemanticTokens) {
- log("Client supports legacy semanticHighlights notification and standard "
- "semanticTokens request, choosing the latter (no notifications).");
- Opts.TheiaSemanticHighlighting = false;
- }
- if (Opts.TheiaSemanticHighlighting) {
- log("Using legacy semanticHighlights notification, which will be removed "
- "in clangd 13. Clients should use the standard semanticTokens "
- "request instead.");
+ !Params.capabilities.SemanticTokens) {
+ elog("Client requested legacy semanticHighlights notification, which is "
+ "no longer supported. Migrate to standard semanticTokens request");
}
if (Params.rootUri && *Params.rootUri)
@@ -674,11 +654,6 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
}}}};
if (Opts.Encoding)
Result["offsetEncoding"] = *Opts.Encoding;
- if (Opts.TheiaSemanticHighlighting)
- Result.getObject("capabilities")
- ->insert(
- {"semanticHighlighting",
- llvm::json::Object{{"scopes", buildHighlightScopeLookupTable()}}});
if (Opts.FoldingRanges)
Result.getObject("capabilities")->insert({"foldingRangeProvider", true});
Reply(std::move(Result));
@@ -898,10 +873,6 @@ void ClangdLSPServer::onDocumentDidClose(
std::lock_guard<std::mutex> Lock(FixItsMutex);
FixItsMap.erase(File);
}
- {
- std::lock_guard<std::mutex> HLock(HighlightingsMutex);
- FileToHighlightings.erase(File);
- }
{
std::lock_guard<std::mutex> HLock(SemanticTokensMutex);
LastSemanticTokens.erase(File);
@@ -1313,11 +1284,6 @@ void ClangdLSPServer::applyConfiguration(
[&](llvm::StringRef File) { return ModifiedFiles.count(File) != 0; });
}
-void ClangdLSPServer::publishTheiaSemanticHighlighting(
- const TheiaSemanticHighlightingParams &Params) {
- notify("textDocument/semanticHighlighting", Params);
-}
-
void ClangdLSPServer::publishDiagnostics(
const PublishDiagnosticsParams &Params) {
notify("textDocument/publishDiagnostics", Params);
@@ -1628,27 +1594,6 @@ bool ClangdLSPServer::shouldRunCompletion(
return allowImplicitCompletion(Code->Contents, *Offset);
}
-void ClangdLSPServer::onHighlightingsReady(
- PathRef File, llvm::StringRef Version,
- std::vector<HighlightingToken> Highlightings) {
- std::vector<HighlightingToken> Old;
- std::vector<HighlightingToken> HighlightingsCopy = Highlightings;
- {
- std::lock_guard<std::mutex> Lock(HighlightingsMutex);
- Old = std::move(FileToHighlightings[File]);
- FileToHighlightings[File] = std::move(HighlightingsCopy);
- }
- // LSP allows us to send incremental edits of highlightings. Also need to
diff
- // to remove highlightings from tokens that should no longer have them.
- std::vector<LineHighlightings> Diffed =
diff Highlightings(Highlightings, Old);
- TheiaSemanticHighlightingParams Notification;
- Notification.TextDocument.uri =
- URIForFile::canonicalize(File, /*TUPath=*/File);
- Notification.TextDocument.version = decodeVersion(Version);
- Notification.Lines = toTheiaSemanticHighlightingInformation(Diffed);
- publishTheiaSemanticHighlighting(Notification);
-}
-
void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version,
std::vector<Diag> Diagnostics) {
PublishDiagnosticsParams Notification;
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index d8ce2dbe53db..d8863a7c67e8 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -84,9 +84,6 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
void onDiagnosticsReady(PathRef File, llvm::StringRef Version,
std::vector<Diag> Diagnostics) override;
void onFileUpdated(PathRef File, const TUStatus &Status) override;
- void
- onHighlightingsReady(PathRef File, llvm::StringRef Version,
- std::vector<HighlightingToken> Highlightings) override;
void onBackgroundIndexProgress(const BackgroundQueue::Stats &Stats) override;
// LSP methods. Notifications have signature void(const Params&).
@@ -179,10 +176,6 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
llvm::function_ref<bool(llvm::StringRef File)> Filter);
void applyConfiguration(const ConfigurationSettings &Settings);
- /// Sends a "publishSemanticHighlighting" notification to the LSP client.
- void
- publishTheiaSemanticHighlighting(const TheiaSemanticHighlightingParams &);
-
/// Sends a "publishDiagnostics" notification to the LSP client.
void publishDiagnostics(const PublishDiagnosticsParams &);
@@ -214,8 +207,6 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
DiagnosticToReplacementMap;
/// Caches FixIts per file and diagnostics
llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
- std::mutex HighlightingsMutex;
- llvm::StringMap<std::vector<HighlightingToken>> FileToHighlightings;
// Last semantic-tokens response, for incremental requests.
std::mutex SemanticTokensMutex;
llvm::StringMap<SemanticTokens> LastSemanticTokens;
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index fd89ec1c45dc..a1ffacf96848 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -65,10 +65,8 @@ namespace {
// Update the FileIndex with new ASTs and plumb the diagnostics responses.
struct UpdateIndexCallbacks : public ParsingCallbacks {
UpdateIndexCallbacks(FileIndex *FIndex,
- ClangdServer::Callbacks *ServerCallbacks,
- bool TheiaSemanticHighlighting)
- : FIndex(FIndex), ServerCallbacks(ServerCallbacks),
- TheiaSemanticHighlighting(TheiaSemanticHighlighting) {}
+ ClangdServer::Callbacks *ServerCallbacks)
+ : FIndex(FIndex), ServerCallbacks(ServerCallbacks) {}
void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
@@ -82,17 +80,10 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
FIndex->updateMain(Path, AST);
std::vector<Diag> Diagnostics = AST.getDiagnostics();
- std::vector<HighlightingToken> Highlightings;
- if (TheiaSemanticHighlighting)
- Highlightings = getSemanticHighlightings(AST);
-
if (ServerCallbacks)
Publish([&]() {
ServerCallbacks->onDiagnosticsReady(Path, AST.version(),
std::move(Diagnostics));
- if (TheiaSemanticHighlighting)
- ServerCallbacks->onHighlightingsReady(Path, AST.version(),
- std::move(Highlightings));
});
}
@@ -111,7 +102,6 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
private:
FileIndex *FIndex;
ClangdServer::Callbacks *ServerCallbacks;
- bool TheiaSemanticHighlighting;
};
} // namespace
@@ -121,7 +111,6 @@ ClangdServer::Options ClangdServer::optsForTest() {
Opts.UpdateDebounce = DebouncePolicy::fixed(/*zero*/ {});
Opts.StorePreamblesInMemory = true;
Opts.AsyncThreadsCount = 4; // Consistent!
- Opts.TheiaSemanticHighlighting = true;
return Opts;
}
@@ -149,8 +138,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
// critical paths.
WorkScheduler(
CDB, TUScheduler::Options(Opts),
- std::make_unique<UpdateIndexCallbacks>(
- DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) {
+ std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), Callbacks)) {
// Adds an index to the stack, at higher priority than existing indexes.
auto AddIndex = [&](SymbolIndex *Idx) {
if (this->Index != nullptr) {
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index e3de7013ba32..c299d7334603 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -68,12 +68,6 @@ class ClangdServer {
/// May be called concurrently for separate files, not for a single file.
virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
- /// Called by ClangdServer when some \p Highlightings for \p File are ready.
- /// May be called concurrently for separate files, not for a single file.
- virtual void
- onHighlightingsReady(PathRef File, llvm::StringRef Version,
- std::vector<HighlightingToken> Highlightings) {}
-
/// Called when background indexing tasks are enqueued/started/completed.
/// Not called concurrently.
virtual void
@@ -145,9 +139,6 @@ class ClangdServer {
/// fetch system include path.
std::vector<std::string> QueryDriverGlobs;
- /// Enable notification-based semantic highlighting.
- bool TheiaSemanticHighlighting = false;
-
/// Enable preview of FoldingRanges feature.
bool FoldingRanges = false;
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index 1543b7c1e09c..a1bf136ff6b0 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -1314,25 +1314,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, OffsetEncoding Enc) {
return OS << toString(Enc);
}
-bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
- const TheiaSemanticHighlightingInformation &Rhs) {
- return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens;
-}
-
-llvm::json::Value
-toJSON(const TheiaSemanticHighlightingInformation &Highlighting) {
- return llvm::json::Object{{"line", Highlighting.Line},
- {"tokens", Highlighting.Tokens},
- {"isInactive", Highlighting.IsInactive}};
-}
-
-llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting) {
- return llvm::json::Object{
- {"textDocument", Highlighting.TextDocument},
- {"lines", std::move(Highlighting.Lines)},
- };
-}
-
bool fromJSON(const llvm::json::Value &Params, SelectionRangeParams &S,
llvm::json::Path P) {
llvm::json::ObjectMapper O(Params, P);
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index a6498088647c..59dc4f209b34 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -449,9 +449,8 @@ struct ClientCapabilities {
bool SemanticTokens = false;
/// Client supports Theia semantic highlighting extension.
/// https://github.com/microsoft/vscode-languageserver-node/pull/367
- /// This will be ignored if the client also supports semanticTokens.
+ /// clangd no longer supports this, we detect it just to log a warning.
/// textDocument.semanticHighlightingCapabilities.semanticHighlighting
- /// FIXME: drop this support once clients support LSP 3.16 Semantic Tokens.
bool TheiaSemanticHighlighting = false;
/// Supported encodings for LSP character offsets. (clangd extension).
@@ -1566,33 +1565,6 @@ struct SemanticTokensOrDelta {
};
llvm::json::Value toJSON(const SemanticTokensOrDelta &);
-/// Represents a semantic highlighting information that has to be applied on a
-/// specific line of the text document.
-struct TheiaSemanticHighlightingInformation {
- /// The line these highlightings belong to.
- int Line = 0;
- /// The base64 encoded string of highlighting tokens.
- std::string Tokens;
- /// Is the line in an inactive preprocessor branch?
- /// This is a clangd extension.
- /// An inactive line can still contain highlighting tokens as well;
- /// clients should combine line style and token style if possible.
- bool IsInactive = false;
-};
-bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
- const TheiaSemanticHighlightingInformation &Rhs);
-llvm::json::Value
-toJSON(const TheiaSemanticHighlightingInformation &Highlighting);
-
-/// Parameters for the semantic highlighting (server-side) push notification.
-struct TheiaSemanticHighlightingParams {
- /// The textdocument these highlightings belong to.
- VersionedTextDocumentIdentifier TextDocument;
- /// The lines of highlightings that should be sent.
- std::vector<TheiaSemanticHighlightingInformation> Lines;
-};
-llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting);
-
struct SelectionRangeParams {
/// The text document.
TextDocumentIdentifier textDocument;
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 0e50fc133e4d..23a58b68ce32 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -526,29 +526,6 @@ class CollectExtraHighlightings
private:
HighlightingsBuilder &H;
};
-
-void write32be(uint32_t I, llvm::raw_ostream &OS) {
- std::array<char, 4> Buf;
- llvm::support::endian::write32be(Buf.data(), I);
- OS.write(Buf.data(), Buf.size());
-}
-
-void write16be(uint16_t I, llvm::raw_ostream &OS) {
- std::array<char, 2> Buf;
- llvm::support::endian::write16be(Buf.data(), I);
- OS.write(Buf.data(), Buf.size());
-}
-
-// Get the highlightings on \c Line where the first entry of line is at \c
-// StartLineIt. If it is not at \c StartLineIt an empty vector is returned.
-ArrayRef<HighlightingToken>
-takeLine(ArrayRef<HighlightingToken> AllTokens,
- ArrayRef<HighlightingToken>::iterator StartLineIt, int Line) {
- return ArrayRef<HighlightingToken>(StartLineIt, AllTokens.end())
- .take_while([Line](const HighlightingToken &Token) {
- return Token.R.start.line == Line;
- });
-}
} // namespace
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
@@ -656,64 +633,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
}
}
-std::vector<LineHighlightings>
-
diff Highlightings(ArrayRef<HighlightingToken> New,
- ArrayRef<HighlightingToken> Old) {
- assert(std::is_sorted(New.begin(), New.end()) &&
- "New must be a sorted vector");
- assert(std::is_sorted(Old.begin(), Old.end()) &&
- "Old must be a sorted vector");
-
- // FIXME: There's an edge case when tokens span multiple lines. If the first
- // token on the line started on a line above the current one and the rest of
- // the line is the equal to the previous one than we will remove all
- // highlights but the ones for the token spanning multiple lines. This means
- // that when we get into the LSP layer the only highlights that will be
- // visible are the ones for the token spanning multiple lines.
- // Example:
- // EndOfMultilineToken Token Token Token
- // If "Token Token Token" don't
diff er from previously the line is
- // incorrectly removed. Suggestion to fix is to separate any multiline tokens
- // into one token for every line it covers. This requires reading from the
- // file buffer to figure out the length of each line though.
- std::vector<LineHighlightings> DiffedLines;
- // ArrayRefs to the current line in the highlightings.
- ArrayRef<HighlightingToken> NewLine(New.begin(),
- /*length*/ static_cast<size_t>(0));
- ArrayRef<HighlightingToken> OldLine(Old.begin(),
- /*length*/ static_cast<size_t>(0));
- auto NewEnd = New.end();
- auto OldEnd = Old.end();
- auto NextLineNumber = [&]() {
- int NextNew = NewLine.end() != NewEnd ? NewLine.end()->R.start.line
- : std::numeric_limits<int>::max();
- int NextOld = OldLine.end() != OldEnd ? OldLine.end()->R.start.line
- : std::numeric_limits<int>::max();
- return std::min(NextNew, NextOld);
- };
-
- for (int LineNumber = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd;
- LineNumber = NextLineNumber()) {
- NewLine = takeLine(New, NewLine.end(), LineNumber);
- OldLine = takeLine(Old, OldLine.end(), LineNumber);
- if (NewLine != OldLine) {
- DiffedLines.push_back({LineNumber, NewLine, /*IsInactive=*/false});
-
- // Turn a HighlightingKind::InactiveCode token into the IsInactive flag.
- auto &AddedLine = DiffedLines.back();
- llvm::erase_if(AddedLine.Tokens, [&](const HighlightingToken &T) {
- if (T.Kind == HighlightingKind::InactiveCode) {
- AddedLine.IsInactive = true;
- return true;
- }
- return false;
- });
- }
- }
-
- return DiffedLines;
-}
-
bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
return std::tie(L.R, L.Kind, L.Modifiers) ==
std::tie(R.R, R.Kind, R.Modifiers);
@@ -722,9 +641,6 @@ bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
return std::tie(L.R, L.Kind, R.Modifiers) <
std::tie(R.R, R.Kind, R.Modifiers);
}
-bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
- return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
-}
std::vector<SemanticToken>
toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
@@ -829,87 +745,6 @@ llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
llvm_unreachable("unhandled HighlightingModifier");
}
-std::vector<TheiaSemanticHighlightingInformation>
-toTheiaSemanticHighlightingInformation(
- llvm::ArrayRef<LineHighlightings> Tokens) {
- if (Tokens.size() == 0)
- return {};
-
- // FIXME: Tokens might be multiple lines long (block comments) in this case
- // this needs to add multiple lines for those tokens.
- std::vector<TheiaSemanticHighlightingInformation> Lines;
- Lines.reserve(Tokens.size());
- for (const auto &Line : Tokens) {
- llvm::SmallVector<char> LineByteTokens;
- llvm::raw_svector_ostream OS(LineByteTokens);
- for (const auto &Token : Line.Tokens) {
- // Writes the token to LineByteTokens in the byte format specified by the
- // LSP proposal. Described below.
- // |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->|
- // | character | length | index |
-
- write32be(Token.R.start.character, OS);
- write16be(Token.R.end.character - Token.R.start.character, OS);
- write16be(static_cast<int>(Token.Kind), OS);
- }
-
- Lines.push_back({Line.Line, encodeBase64(LineByteTokens), Line.IsInactive});
- }
-
- return Lines;
-}
-
-llvm::StringRef toTextMateScope(HighlightingKind Kind) {
- // FIXME: Add scopes for C and Objective C.
- switch (Kind) {
- case HighlightingKind::Function:
- return "entity.name.function.cpp";
- case HighlightingKind::Method:
- return "entity.name.function.method.cpp";
- case HighlightingKind::StaticMethod:
- return "entity.name.function.method.static.cpp";
- case HighlightingKind::Variable:
- return "variable.other.cpp";
- case HighlightingKind::LocalVariable:
- return "variable.other.local.cpp";
- case HighlightingKind::Parameter:
- return "variable.parameter.cpp";
- case HighlightingKind::Field:
- return "variable.other.field.cpp";
- case HighlightingKind::StaticField:
- return "variable.other.field.static.cpp";
- case HighlightingKind::Class:
- return "entity.name.type.class.cpp";
- case HighlightingKind::Enum:
- return "entity.name.type.enum.cpp";
- case HighlightingKind::EnumConstant:
- return "variable.other.enummember.cpp";
- case HighlightingKind::Typedef:
- return "entity.name.type.typedef.cpp";
- case HighlightingKind::Type:
- // Fragile: all paths emitting `Type` are dependent names for now.
- // But toTextMateScope is going away soon.
- return "entity.name.type.dependent.cpp";
- case HighlightingKind::Unknown:
- // Fragile: all paths emitting `Unknown` are dependent names for now.
- // But toTextMateScope is going away soon.
- return "entity.name.other.dependent.cpp";
- case HighlightingKind::Namespace:
- return "entity.name.namespace.cpp";
- case HighlightingKind::TemplateParameter:
- return "entity.name.type.template.cpp";
- case HighlightingKind::Concept:
- return "entity.name.type.concept.cpp";
- case HighlightingKind::Primitive:
- return "storage.type.primitive.cpp";
- case HighlightingKind::Macro:
- return "entity.name.function.preprocessor.cpp";
- case HighlightingKind::InactiveCode:
- return "meta.disabled";
- }
- llvm_unreachable("unhandled HighlightingKind");
-}
-
std::vector<SemanticTokensEdit>
diff Tokens(llvm::ArrayRef<SemanticToken> Old,
llvm::ArrayRef<SemanticToken> New) {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h
index a34806d4a822..e4c36ab5261e 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.h
+++ b/clang-tools-extra/clangd/SemanticHighlighting.h
@@ -8,19 +8,9 @@
//
// This file supports semantic highlighting: categorizing tokens in the file so
// that the editor can color/style them
diff erently.
-//
// This is particularly valuable for C++: its complex and context-dependent
// grammar is a challenge for simple syntax-highlighting techniques.
//
-// We support two protocols for providing highlights to the client:
-// - the `textDocument/semanticTokens` request from LSP 3.16
-// https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts
-// - the earlier proposed `textDocument/semanticHighlighting` notification
-// https://github.com/microsoft/vscode-languageserver-node/pull/367
-// This is referred to as "Theia" semantic highlighting in the code.
-// It was supported from clangd 9 but should be considered deprecated as of
-// clangd 11 and eventually removed.
-//
// Semantic highlightings are calculated for an AST by visiting every AST node
// and classifying nodes that are interesting to highlight (variables/function
// calls etc.).
@@ -103,15 +93,6 @@ struct HighlightingToken {
bool operator==(const HighlightingToken &L, const HighlightingToken &R);
bool operator<(const HighlightingToken &L, const HighlightingToken &R);
-/// Contains all information about highlightings on a single line.
-struct LineHighlightings {
- int Line;
- std::vector<HighlightingToken> Tokens;
- bool IsInactive;
-};
-
-bool operator==(const LineHighlightings &L, const LineHighlightings &R);
-
// Returns all HighlightingTokens from an AST. Only generates highlights for the
// main AST.
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
@@ -122,28 +103,6 @@ llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier);
std::vector<SemanticTokensEdit>
diff Tokens(llvm::ArrayRef<SemanticToken> Before,
llvm::ArrayRef<SemanticToken> After);
-/// Converts a HighlightingKind to a corresponding TextMate scope
-/// (https://manual.macromates.com/en/language_grammars).
-llvm::StringRef toTextMateScope(HighlightingKind Kind);
-
-/// Convert to LSP's semantic highlighting information.
-std::vector<TheiaSemanticHighlightingInformation>
-toTheiaSemanticHighlightingInformation(
- llvm::ArrayRef<LineHighlightings> Tokens);
-
-/// Return a line-by-line
diff between two highlightings.
-/// - if the tokens on a line are the same in both highlightings, this line is
-/// omitted.
-/// - if a line exists in New but not in Old, the tokens on this line are
-/// emitted.
-/// - if a line does not exist in New but exists in Old, an empty line is
-/// emitted (to tell client to clear the previous highlightings on this line).
-///
-/// REQUIRED: Old and New are sorted.
-std::vector<LineHighlightings>
-
diff Highlightings(ArrayRef<HighlightingToken> New,
- ArrayRef<HighlightingToken> Old);
-
} // namespace clangd
} // namespace clang
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index 26bf430db46f..01b583fe64f3 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -513,7 +513,7 @@ class ASTWorker {
/// PreambleRequest.
mutable std::condition_variable PreambleCV;
/// Guards the callback that publishes results of AST-related computations
- /// (diagnostics, highlightings) and file statuses.
+ /// (diagnostics) and file statuses.
std::mutex PublishMu;
// Used to prevent remove document + add document races that lead to
// out-of-order callbacks for publishing results of onMainAST callback.
diff --git a/clang-tools-extra/clangd/TUScheduler.h b/clang-tools-extra/clangd/TUScheduler.h
index 095cd0b41ca5..2c381923a37e 100644
--- a/clang-tools-extra/clangd/TUScheduler.h
+++ b/clang-tools-extra/clangd/TUScheduler.h
@@ -151,11 +151,11 @@ class ParsingCallbacks {
/// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
/// optimal performance.
///
- /// When information about the file (diagnostics, syntax highlighting) is
+ /// When information about the file (e.g. diagnostics) is
/// published to clients, this should be wrapped in Publish, e.g.
/// void onMainAST(...) {
- /// Highlights = computeHighlights();
- /// Publish([&] { notifyHighlights(Path, Highlights); });
+ /// Diags = renderDiagnostics();
+ /// Publish([&] { notifyDiagnostics(Path, Diags); });
/// }
/// This guarantees that clients will see results in the correct sequence if
/// the file is concurrently closed and/or reopened. (The lambda passed to
diff --git a/clang-tools-extra/clangd/test/semantic-highlighting.test b/clang-tools-extra/clangd/test/semantic-highlighting.test
deleted file mode 100644
index bca6b373aa22..000000000000
--- a/clang-tools-extra/clangd/test/semantic-highlighting.test
+++ /dev/null
@@ -1,145 +0,0 @@
-# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
-{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
----
-# CHECK: "id": 0,
-# CHECK: "semanticHighlighting": {
-# CHECK-NEXT: "scopes": [
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.other.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.other.local.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.parameter.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.function.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.function.method.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.function.method.static.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.other.field.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.other.field.static.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.class.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.enum.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "variable.other.enummember.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.typedef.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.dependent.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.other.dependent.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.namespace.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.template.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.type.concept.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "storage.type.primitive.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "entity.name.function.preprocessor.cpp"
-# CHECK-NEXT: ],
-# CHECK-NEXT: [
-# CHECK-NEXT: "meta.disabled"
-# CHECK-NEXT: ]
-# CHECK-NEXT: ]
-# CHECK-NEXT: },
----
-{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}}
-# CHECK: "method": "textDocument/semanticHighlighting",
-# CHECK-NEXT: "params": {
-# CHECK-NEXT: "lines": [
-# CHECK-NEXT: {
-# CHECK-NEXT: "isInactive": false,
-# CHECK-NEXT: "line": 0,
-# CHECK-NEXT: "tokens": "AAAABAABAAA="
-# CHECK-NEXT: }
-# CHECK-NEXT: ],
-# CHECK-NEXT: "textDocument": {
-# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
-# CHECK-NEXT: "version": 0
-# CHECK-NEXT: }
-# CHECK-NEXT: }
-# CHECK-NEXT:}
----
-{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","text":"int x = 2;\nint y = 2;"}}}
-# CHECK: "method": "textDocument/semanticHighlighting",
-# CHECK-NEXT: "params": {
-# CHECK-NEXT: "lines": [
-# CHECK-NEXT: {
-# CHECK-NEXT: "isInactive": false,
-# CHECK-NEXT: "line": 0,
-# CHECK-NEXT: "tokens": "AAAABAABAAA="
-# CHECK-NEXT: }
-# CHECK-NEXT: {
-# CHECK-NEXT: "isInactive": false,
-# CHECK-NEXT: "line": 1,
-# CHECK-NEXT: "tokens": "AAAABAABAAA="
-# CHECK-NEXT: }
-# CHECK-NEXT: ],
-# CHECK-NEXT: "textDocument": {
-# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo2.cpp",
-# CHECK-NEXT: "version": 0
-# CHECK-NEXT: }
-# CHECK-NEXT: }
-# CHECK-NEXT:}
----
-{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp"},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
-# CHECK: "method": "textDocument/semanticHighlighting",
-# CHECK-NEXT: "params": {
-# CHECK-NEXT: "lines": [
-# CHECK-NEXT: {
-# CHECK-NEXT: "isInactive": false,
-# CHECK-NEXT: "line": 1,
-# CHECK-NEXT: "tokens": "AAAABAABAAA="
-# CHECK-NEXT: }
-# CHECK-NEXT: ],
-# CHECK-NEXT: "textDocument": {
-# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
-# CHECK-NEXT: "version": 1
-# CHECK-NEXT: }
-# CHECK-NEXT: }
-# CHECK-NEXT:}
----
-{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp"},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 1,"character": 10}},"rangeLength": 11,"text": ""}]}}
-# CHECK: "method": "textDocument/semanticHighlighting",
-# CHECK-NEXT: "params": {
-# CHECK-NEXT: "lines": [
-# CHECK-NEXT: {
-# CHECK-NEXT: "isInactive": false,
-# CHECK-NEXT: "line": 1,
-# CHECK-NEXT: "tokens": ""
-# CHECK-NEXT: }
-# CHECK-NEXT: ],
-# CHECK-NEXT: "textDocument": {
-# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
-# CHECK-NEXT: "version": 2
-# CHECK-NEXT: }
-# CHECK-NEXT: }
-# CHECK-NEXT:}
----
-{"jsonrpc":"2.0","id":3,"method":"shutdown"}
----
-{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/test/semantic-tokens.test b/clang-tools-extra/clangd/test/semantic-tokens.test
index dc79c79b6e76..5cfe6489b0c4 100644
--- a/clang-tools-extra/clangd/test/semantic-tokens.test
+++ b/clang-tools-extra/clangd/test/semantic-tokens.test
@@ -1,6 +1,6 @@
# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s -implicit-check-not=semanticHighlight
# Send capabilities for both Theia semanticHighlight & standard semanticTokens.
-# clangd should not use/acknowledge the Theia protocol in this case.
+# clangd should not use/acknowledge the Theia protocol.
{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{"textDocument":{
"semanticHighlightingCapabilities":{"semanticHighlighting":true},
"semanticTokens":{"dynamicRegistration":true}
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index 577b01d3eae8..917248628d49 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -29,51 +29,6 @@ namespace {
using testing::IsEmpty;
using testing::SizeIs;
-MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
-MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
-
-std::vector<HighlightingToken>
-makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
- std::vector<HighlightingToken> Tokens(Ranges.size());
- for (int I = 0, End = Ranges.size(); I < End; ++I) {
- Tokens[I].R = Ranges[I];
- Tokens[I].Kind = Kind;
- }
-
- return Tokens;
-}
-
-std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
- static const std::map<HighlightingKind, std::string> KindToString{
- {HighlightingKind::Variable, "Variable"},
- {HighlightingKind::LocalVariable, "LocalVariable"},
- {HighlightingKind::Parameter, "Parameter"},
- {HighlightingKind::Function, "Function"},
- {HighlightingKind::Class, "Class"},
- {HighlightingKind::Enum, "Enum"},
- {HighlightingKind::Namespace, "Namespace"},
- {HighlightingKind::EnumConstant, "EnumConstant"},
- {HighlightingKind::Field, "Field"},
- {HighlightingKind::StaticField, "StaticField"},
- {HighlightingKind::Method, "Method"},
- {HighlightingKind::StaticMethod, "StaticMethod"},
- {HighlightingKind::Typedef, "Typedef"},
- {HighlightingKind::Type, "Type"},
- {HighlightingKind::Unknown, "Unknown"},
- {HighlightingKind::TemplateParameter, "TemplateParameter"},
- {HighlightingKind::Concept, "Concept"},
- {HighlightingKind::Primitive, "Primitive"},
- {HighlightingKind::Macro, "Macro"}};
- std::vector<HighlightingToken> ExpectedTokens;
- for (const auto &KindString : KindToString) {
- std::vector<HighlightingToken> Toks = makeHighlightingTokens(
- Test.ranges(KindString.second), KindString.first);
- ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
- }
- llvm::sort(ExpectedTokens);
- return ExpectedTokens;
-}
-
/// Annotates the input code with provided semantic highlightings. Results look
/// something like:
/// class $Class[[X]] {
@@ -134,39 +89,6 @@ void checkHighlightings(llvm::StringRef Code,
EXPECT_EQ(Code, annotate(Test.code(), Actual));
}
-// Any annotations in OldCode and NewCode are converted into their corresponding
-// HighlightingToken. The tokens are
diff ed against each other. Any lines where
-// the tokens should
diff must be marked with a ^ somewhere on that line in
-// NewCode. If there are
diff s that aren't marked with ^ the test fails. The
-// test also fails if there are lines marked with ^ that don't
diff er.
-void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
- Annotations OldTest(OldCode);
- Annotations NewTest(NewCode);
- std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
- std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
-
- llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
- for (const Position &Point : NewTest.points()) {
- ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
- // are inserted on these lines later.
- }
- std::vector<LineHighlightings> ExpectedLinePairHighlighting;
- for (const HighlightingToken &Token : NewTokens) {
- auto It = ExpectedLines.find(Token.R.start.line);
- if (It != ExpectedLines.end())
- It->second.push_back(Token);
- }
- for (auto &LineTokens : ExpectedLines)
- ExpectedLinePairHighlighting.push_back(
- {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
-
- std::vector<LineHighlightings> ActualDiffed =
-
diff Highlightings(NewTokens, OldTokens);
- EXPECT_THAT(ActualDiffed,
- testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
- << OldCode;
-}
-
constexpr static uint32_t ScopeModifierMask =
1 << unsigned(HighlightingModifier::FunctionScope) |
1 << unsigned(HighlightingModifier::ClassScope) |
@@ -809,30 +731,6 @@ TEST(SemanticHighlighting, ScopeModifiers) {
checkHighlightings(Test, {}, ScopeModifierMask);
}
-TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
- class HighlightingsCounter : public ClangdServer::Callbacks {
- public:
- std::atomic<int> Count = {0};
-
- void onHighlightingsReady(
- PathRef File, llvm::StringRef Version,
- std::vector<HighlightingToken> Highlightings) override {
- ++Count;
- }
- };
-
- auto FooCpp = testPath("foo.cpp");
- MockFS FS;
- FS.Files[FooCpp] = "";
-
- MockCompilationDatabase MCD;
- HighlightingsCounter Counter;
- ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
- Server.addDocument(FooCpp, "int a;");
- ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
- ASSERT_EQ(Counter.Count, 1);
-}
-
// Ranges are highlighted as variables, unless highlighted as $Function etc.
std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
Annotations A(MarkedText);
@@ -912,149 +810,6 @@ TEST(SemanticHighlighting,
diff SemanticTokens) {
EXPECT_EQ(3u, Diff.front().tokens[2].length);
}
-TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
- auto CreatePosition = [](int Line, int Character) -> Position {
- Position Pos;
- Pos.line = Line;
- Pos.character = Character;
- return Pos;
- };
-
- std::vector<LineHighlightings> Tokens{
- {3,
- {{HighlightingKind::Variable, 0,
- Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
- {HighlightingKind::Function, 0,
- Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
- /* IsInactive = */ false},
- {1,
- {{HighlightingKind::Variable, 0,
- Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
- /* IsInactive = */ true}};
- std::vector<TheiaSemanticHighlightingInformation> ActualResults =
- toTheiaSemanticHighlightingInformation(Tokens);
- std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
- {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
- EXPECT_EQ(ActualResults, ExpectedResults);
-}
-
-TEST(SemanticHighlighting, HighlightingDiffer) {
- struct {
- llvm::StringRef OldCode;
- llvm::StringRef NewCode;
- } TestCases[]{{
- R"(
- $Variable[[A]]
- $Class[[B]]
- $Function[[C]]
- )",
- R"(
- $Variable[[A]]
- $Class[[D]]
- $Function[[C]]
- )"},
- {
- R"(
- $Class[[C]]
- $Field[[F]]
- $Variable[[V]]
- $Class[[C]] $Variable[[V]] $Field[[F]]
- )",
- R"(
- $Class[[C]]
- $Field[[F]]
- ^$Function[[F]]
- $Class[[C]] $Variable[[V]] $Field[[F]]
- )"},
- {
- R"(
-
- $Class[[A]]
- $Variable[[A]]
- )",
- R"(
-
- ^
- ^$Class[[A]]
- ^$Variable[[A]]
- )"},
- {
- R"(
- $Class[[C]]
- $Field[[F]]
- $Variable[[V]]
- $Class[[C]] $Variable[[V]] $Field[[F]]
- )",
- R"(
- $Class[[C]]
- ^
- ^
- $Class[[C]] $Variable[[V]] $Field[[F]]
- )"},
- {
- R"(
- $Class[[A]]
- $Variable[[A]]
- $Variable[[A]]
- )",
- R"(
- $Class[[A]]
- ^$Variable[[AA]]
- $Variable[[A]]
- )"},
- {
- R"(
- $Class[[A]]
- $Variable[[A]]
- )",
- R"(
- $Class[[A]]
- $Variable[[A]]
- ^$Class[[A]]
- ^$Variable[[A]]
- )"},
- {
- R"(
- $Variable[[A]]
- $Variable[[A]]
- $Variable[[A]]
- )",
- R"(
- ^$Class[[A]]
- ^$Class[[A]]
- ^$Class[[A]]
- )"}};
-
- for (const auto &Test : TestCases)
- checkDiffedHighlights(Test.OldCode, Test.NewCode);
-}
-
-TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
- llvm::StringRef OldCode =
- R"(
- $Class[[A]]
- $Variable[[A]]
- $Class[[A]]
- $Variable[[A]]
- )";
- llvm::StringRef NewCode =
- R"(
- $Class[[A]] // line 1
- $Variable[[A]] // line 2
- )";
-
- Annotations OldTest(OldCode);
- Annotations NewTest(NewCode);
- std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
- std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
-
- auto ActualDiff =
diff Highlightings(NewTokens, OldTokens);
- EXPECT_THAT(ActualDiff,
- testing::UnorderedElementsAre(
- testing::AllOf(LineNumber(3), EmptyHighlightings()),
- testing::AllOf(LineNumber(4), EmptyHighlightings())));
-}
-
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list