[clang-tools-extra] 6feaa54 - [clangd] Implement configs to stop clangd produce a certain semantic tokens
Nathan Ridge via cfe-commits
cfe-commits at lists.llvm.org
Fri May 26 00:25:43 PDT 2023
Author: Qingyuan Zheng
Date: 2023-05-26T03:25:35-04:00
New Revision: 6feaa5416bf63f9609478d4458485c7306506e26
URL: https://github.com/llvm/llvm-project/commit/6feaa5416bf63f9609478d4458485c7306506e26
DIFF: https://github.com/llvm/llvm-project/commit/6feaa5416bf63f9609478d4458485c7306506e26.diff
LOG: [clangd] Implement configs to stop clangd produce a certain semantic tokens
This patch introduces the following configurations to .clangd:
```
SemanticTokens:
DisabledKinds: [ ... ]
DisabledModifiers: [ ... ]
```
Based on the config, clangd would stop producing a certain type of semantic tokens from the source file.
Fixes https://github.com/clangd/clangd/discussions/1598
Reviewed By: nridge
Differential Revision: https://reviews.llvm.org/D148489
Added:
Modified:
clang-tools-extra/clangd/Config.h
clang-tools-extra/clangd/ConfigCompile.cpp
clang-tools-extra/clangd/ConfigFragment.h
clang-tools-extra/clangd/ConfigYAML.cpp
clang-tools-extra/clangd/SemanticHighlighting.cpp
clang-tools-extra/clangd/SemanticHighlighting.h
clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index 15b4b7ef06fef..4c6fad25384a8 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -150,6 +150,13 @@ struct Config {
// Limit the length of type names in inlay hints. (0 means no limit)
uint32_t TypeNameLimit = 32;
} InlayHints;
+
+ struct {
+ /// Controls highlighting kinds that are disabled.
+ std::vector<std::string> DisabledKinds;
+ /// Controls highlighting modifiers that are disabled.
+ std::vector<std::string> DisabledModifiers;
+ } SemanticTokens;
};
} // namespace clangd
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index 9bd067666f5f8..832dab54febaf 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -196,6 +196,7 @@ struct FragmentCompiler {
compile(std::move(F.Completion));
compile(std::move(F.Hover));
compile(std::move(F.InlayHints));
+ compile(std::move(F.SemanticTokens));
compile(std::move(F.Style));
}
@@ -618,6 +619,37 @@ struct FragmentCompiler {
});
}
+ void compile(Fragment::SemanticTokensBlock &&F) {
+ if (!F.DisabledKinds.empty()) {
+ std::vector<std::string> DisabledKinds;
+ for (auto &Kind : F.DisabledKinds)
+ DisabledKinds.push_back(std::move(*Kind));
+
+ Out.Apply.push_back(
+ [DisabledKinds(std::move(DisabledKinds))](const Params &, Config &C) {
+ for (auto &Kind : DisabledKinds) {
+ auto It = llvm::find(C.SemanticTokens.DisabledKinds, Kind);
+ if (It == C.SemanticTokens.DisabledKinds.end())
+ C.SemanticTokens.DisabledKinds.push_back(std::move(Kind));
+ }
+ });
+ }
+ if (!F.DisabledModifiers.empty()) {
+ std::vector<std::string> DisabledModifiers;
+ for (auto &Kind : F.DisabledModifiers)
+ DisabledModifiers.push_back(std::move(*Kind));
+
+ Out.Apply.push_back([DisabledModifiers(std::move(DisabledModifiers))](
+ const Params &, Config &C) {
+ for (auto &Kind : DisabledModifiers) {
+ auto It = llvm::find(C.SemanticTokens.DisabledModifiers, Kind);
+ if (It == C.SemanticTokens.DisabledModifiers.end())
+ C.SemanticTokens.DisabledModifiers.push_back(std::move(Kind));
+ }
+ });
+ }
+ }
+
constexpr static llvm::SourceMgr::DiagKind Error = llvm::SourceMgr::DK_Error;
constexpr static llvm::SourceMgr::DiagKind Warning =
llvm::SourceMgr::DK_Warning;
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index c1bac1019641d..fe2f68f5c45bc 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -235,7 +235,6 @@ struct Fragment {
/// - std::nullopt
std::optional<Located<std::string>> UnusedIncludes;
-
/// Enable emitting diagnostics using stale preambles.
std::optional<Located<bool>> AllowStalePreamble;
@@ -326,6 +325,15 @@ struct Fragment {
std::optional<Located<uint32_t>> TypeNameLimit;
};
InlayHintsBlock InlayHints;
+
+ /// Configures semantic tokens that are produced by clangd.
+ struct SemanticTokensBlock {
+ /// Disables clangd to produce semantic tokens for the given kinds.
+ std::vector<Located<std::string>> DisabledKinds;
+ /// Disables clangd to assign semantic tokens with the given modifiers.
+ std::vector<Located<std::string>> DisabledModifiers;
+ };
+ SemanticTokensBlock SemanticTokens;
};
} // namespace config
diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp
index d16860a1ccf48..4d91062d890fe 100644
--- a/clang-tools-extra/clangd/ConfigYAML.cpp
+++ b/clang-tools-extra/clangd/ConfigYAML.cpp
@@ -68,6 +68,7 @@ class Parser {
Dict.handle("Completion", [&](Node &N) { parse(F.Completion, N); });
Dict.handle("Hover", [&](Node &N) { parse(F.Hover, N); });
Dict.handle("InlayHints", [&](Node &N) { parse(F.InlayHints, N); });
+ Dict.handle("SemanticTokens", [&](Node &N) { parse(F.SemanticTokens, N); });
Dict.parse(N);
return !(N.failed() || HadError);
}
@@ -261,6 +262,19 @@ class Parser {
Dict.parse(N);
}
+ void parse(Fragment::SemanticTokensBlock &F, Node &N) {
+ DictParser Dict("SemanticTokens", this);
+ Dict.handle("DisabledKinds", [&](Node &N) {
+ if (auto Values = scalarValues(N))
+ F.DisabledKinds = std::move(*Values);
+ });
+ Dict.handle("DisabledModifiers", [&](Node &N) {
+ if (auto Values = scalarValues(N))
+ F.DisabledModifiers = std::move(*Values);
+ });
+ Dict.parse(N);
+ }
+
// Helper for parsing mapping nodes (dictionaries).
// We don't use YamlIO as we want to control over unknown keys.
class DictParser {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index f884a6b019231..34a0214b082bd 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "SemanticHighlighting.h"
+#include "Config.h"
#include "FindTarget.h"
#include "HeuristicResolver.h"
#include "ParsedAST.h"
@@ -354,13 +355,57 @@ resolveConflict(ArrayRef<HighlightingToken> Tokens) {
return Winner;
}
+/// Filter to remove particular kinds of highlighting tokens and modifiers from
+/// the output.
+class HighlightingFilter {
+public:
+ HighlightingFilter() {
+ for (auto &Active : ActiveKindLookup)
+ Active = true;
+
+ ActiveModifiersMask = ~0;
+ }
+
+ void disableKind(HighlightingKind Kind) {
+ ActiveKindLookup[static_cast<size_t>(Kind)] = false;
+ }
+
+ void disableModifier(HighlightingModifier Modifier) {
+ ActiveModifiersMask &= ~(1 << static_cast<uint32_t>(Modifier));
+ }
+
+ bool isHighlightKindActive(HighlightingKind Kind) const {
+ return ActiveKindLookup[static_cast<size_t>(Kind)];
+ }
+
+ uint32_t maskModifiers(uint32_t Modifiers) const {
+ return Modifiers & ActiveModifiersMask;
+ }
+
+ static HighlightingFilter fromCurrentConfig() {
+ const Config &C = Config::current();
+ HighlightingFilter Filter;
+ for (const auto &Kind : C.SemanticTokens.DisabledKinds)
+ if (auto K = highlightingKindFromString(Kind))
+ Filter.disableKind(*K);
+ for (const auto &Modifier : C.SemanticTokens.DisabledModifiers)
+ if (auto M = highlightingModifierFromString(Modifier))
+ Filter.disableModifier(*M);
+
+ return Filter;
+ }
+
+private:
+ bool ActiveKindLookup[static_cast<size_t>(HighlightingKind::LastKind) + 1];
+ uint32_t ActiveModifiersMask;
+};
+
/// Consumes source locations and maps them to text ranges for highlightings.
class HighlightingsBuilder {
public:
- HighlightingsBuilder(const ParsedAST &AST, bool IncludeInactiveRegionTokens)
+ HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter)
: TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
- LangOpts(AST.getLangOpts()),
- IncludeInactiveRegionTokens(IncludeInactiveRegionTokens) {}
+ LangOpts(AST.getLangOpts()), Filter(Filter) {}
HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
auto Range = getRangeForSourceLocation(Loc);
@@ -412,6 +457,9 @@ class HighlightingsBuilder {
}
HighlightingToken &addToken(Range R, HighlightingKind Kind) {
+ if (!Filter.isHighlightKindActive(Kind))
+ return InvalidHighlightingToken;
+
HighlightingToken HT;
HT.R = std::move(R);
HT.Kind = Kind;
@@ -452,6 +500,7 @@ class HighlightingsBuilder {
}
}
+ Resolved->Modifiers = Filter.maskModifiers(Resolved->Modifiers);
NonConflicting.push_back(*Resolved);
}
// TokRef[Conflicting.size()] is the next token with a
diff erent range (or
@@ -459,7 +508,7 @@ class HighlightingsBuilder {
TokRef = TokRef.drop_front(Conflicting.size());
}
- if (!IncludeInactiveRegionTokens)
+ if (!Filter.isHighlightKindActive(HighlightingKind::InactiveCode))
return NonConflicting;
const auto &SM = AST.getSourceManager();
@@ -535,7 +584,7 @@ class HighlightingsBuilder {
const syntax::TokenBuffer &TB;
const SourceManager &SourceMgr;
const LangOptions &LangOpts;
- bool IncludeInactiveRegionTokens;
+ HighlightingFilter Filter;
std::vector<HighlightingToken> Tokens;
std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
const HeuristicResolver *Resolver = nullptr;
@@ -1104,8 +1153,11 @@ class CollectExtraHighlightings
std::vector<HighlightingToken>
getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {
auto &C = AST.getASTContext();
+ HighlightingFilter Filter = HighlightingFilter::fromCurrentConfig();
+ if (!IncludeInactiveRegionTokens)
+ Filter.disableKind(HighlightingKind::InactiveCode);
// Add highlightings for AST nodes.
- HighlightingsBuilder Builder(AST, IncludeInactiveRegionTokens);
+ HighlightingsBuilder Builder(AST, Filter);
// Highlight 'decltype' and 'auto' as their underlying types.
CollectExtraHighlightings(Builder).TraverseAST(C);
// Highlight all decls and references coming from the AST.
@@ -1224,6 +1276,38 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
}
llvm_unreachable("invalid HighlightingKind");
}
+std::optional<HighlightingKind>
+highlightingKindFromString(llvm::StringRef Name) {
+ static llvm::StringMap<HighlightingKind> Lookup = {
+ {"Variable", HighlightingKind::Variable},
+ {"LocalVariable", HighlightingKind::LocalVariable},
+ {"Parameter", HighlightingKind::Parameter},
+ {"Function", HighlightingKind::Function},
+ {"Method", HighlightingKind::Method},
+ {"StaticMethod", HighlightingKind::StaticMethod},
+ {"Field", HighlightingKind::Field},
+ {"StaticField", HighlightingKind::StaticField},
+ {"Class", HighlightingKind::Class},
+ {"Interface", HighlightingKind::Interface},
+ {"Enum", HighlightingKind::Enum},
+ {"EnumConstant", HighlightingKind::EnumConstant},
+ {"Typedef", HighlightingKind::Typedef},
+ {"Type", HighlightingKind::Type},
+ {"Unknown", HighlightingKind::Unknown},
+ {"Namespace", HighlightingKind::Namespace},
+ {"TemplateParameter", HighlightingKind::TemplateParameter},
+ {"Concept", HighlightingKind::Concept},
+ {"Primitive", HighlightingKind::Primitive},
+ {"Macro", HighlightingKind::Macro},
+ {"Modifier", HighlightingKind::Modifier},
+ {"Operator", HighlightingKind::Operator},
+ {"Bracket", HighlightingKind::Bracket},
+ {"InactiveCode", HighlightingKind::InactiveCode},
+ };
+
+ auto It = Lookup.find(Name);
+ return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
+}
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
switch (K) {
case HighlightingModifier::Declaration:
@@ -1236,6 +1320,33 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
return OS << toSemanticTokenModifier(K);
}
}
+std::optional<HighlightingModifier>
+highlightingModifierFromString(llvm::StringRef Name) {
+ static llvm::StringMap<HighlightingModifier> Lookup = {
+ {"Declaration", HighlightingModifier::Declaration},
+ {"Definition", HighlightingModifier::Definition},
+ {"Deprecated", HighlightingModifier::Deprecated},
+ {"Deduced", HighlightingModifier::Deduced},
+ {"Readonly", HighlightingModifier::Readonly},
+ {"Static", HighlightingModifier::Static},
+ {"Abstract", HighlightingModifier::Abstract},
+ {"Virtual", HighlightingModifier::Virtual},
+ {"DependentName", HighlightingModifier::DependentName},
+ {"DefaultLibrary", HighlightingModifier::DefaultLibrary},
+ {"UsedAsMutableReference", HighlightingModifier::UsedAsMutableReference},
+ {"UsedAsMutablePointer", HighlightingModifier::UsedAsMutablePointer},
+ {"ConstructorOrDestructor",
+ HighlightingModifier::ConstructorOrDestructor},
+ {"UserDefined", HighlightingModifier::UserDefined},
+ {"FunctionScope", HighlightingModifier::FunctionScope},
+ {"ClassScope", HighlightingModifier::ClassScope},
+ {"FileScope", HighlightingModifier::FileScope},
+ {"GlobalScope", HighlightingModifier::GlobalScope},
+ };
+
+ auto It = Lookup.find(Name);
+ return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
+}
bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
return std::tie(L.R, L.Kind, L.Modifiers) ==
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h
index 10dd1193001bb..ca90230dfb8fb 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.h
+++ b/clang-tools-extra/clangd/SemanticHighlighting.h
@@ -61,6 +61,8 @@ enum class HighlightingKind {
};
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K);
+std::optional<HighlightingKind>
+highlightingKindFromString(llvm::StringRef Name);
enum class HighlightingModifier {
Declaration,
@@ -88,6 +90,8 @@ enum class HighlightingModifier {
static_assert(static_cast<unsigned>(HighlightingModifier::LastModifier) < 32,
"Increase width of modifiers bitfield!");
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K);
+std::optional<HighlightingModifier>
+highlightingModifierFromString(llvm::StringRef Name);
// Contains all information needed for the highlighting a token.
struct HighlightingToken {
diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
index 7ed66c67d99f7..dda81ec78aa6b 100644
--- a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
@@ -246,6 +246,23 @@ TEST(ParseYAML, InlayHints) {
EXPECT_EQ(Results[0].InlayHints.DeducedTypes, std::nullopt);
}
+TEST(ParseYAML, SemanticTokens) {
+ CapturedDiags Diags;
+ Annotations YAML(R"yaml(
+SemanticTokens:
+ DisabledKinds: [ Operator, InactiveCode]
+ DisabledModifiers: Readonly
+ )yaml");
+ auto Results =
+ Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
+ ASSERT_THAT(Diags.Diagnostics, IsEmpty());
+ ASSERT_EQ(Results.size(), 1u);
+ EXPECT_THAT(Results[0].SemanticTokens.DisabledKinds,
+ ElementsAre(val("Operator"), val("InactiveCode")));
+ EXPECT_THAT(Results[0].SemanticTokens.DisabledModifiers,
+ ElementsAre(val("Readonly")));
+}
+
TEST(ParseYAML, IncludesIgnoreHeader) {
CapturedDiags Diags;
Annotations YAML(R"yaml(
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index cd912bc220345..c25dff810b764 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "Annotations.h"
+#include "Config.h"
#include "Protocol.h"
#include "SemanticHighlighting.h"
#include "SourceCode.h"
@@ -1260,6 +1261,17 @@ o]] [[bar]])cpp";
EXPECT_EQ(Toks[3].deltaStart, 2u);
EXPECT_EQ(Toks[3].length, 3u);
}
+
+TEST(SemanticHighlighting, WithHighlightingFilter) {
+ llvm::StringRef AnnotatedCode = R"cpp(
+int *$Variable[[x]] = new int;
+)cpp";
+ Config Cfg;
+ Cfg.SemanticTokens.DisabledKinds = {"Operator"};
+ Cfg.SemanticTokens.DisabledModifiers = {"Declaration", "Definition"};
+ WithContextValue WithCfg(Config::Key, std::move(Cfg));
+ checkHighlightings(AnnotatedCode, {}, ~ScopeModifierMask);
+}
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list