[clang] [clang-tools-extra] [clangd] Handle C++20 annot_module_name token (PR #187858)
via cfe-commits
cfe-commits at lists.llvm.org
Sat Mar 21 21:12:33 PDT 2026
https://github.com/yronglin updated https://github.com/llvm/llvm-project/pull/187858
>From 6369995b6ac58ac8cd72eeace7e96d5f30f1c3bd Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Sun, 22 Mar 2026 12:12:12 +0800
Subject: [PATCH] [clangd] Handle C++20 annot_module_name token
Signed-off-by: yronglin <yronglin777 at gmail.com>
---
.../clangd/test/non-existent.test | 82 +++++++++++++++++++
clang/lib/Tooling/Syntax/Tokens.cpp | 23 +++++-
clang/unittests/Tooling/Syntax/TokensTest.cpp | 36 +++++++-
3 files changed, 135 insertions(+), 6 deletions(-)
create mode 100644 clang-tools-extra/clangd/test/non-existent.test
diff --git a/clang-tools-extra/clangd/test/non-existent.test b/clang-tools-extra/clangd/test/non-existent.test
new file mode 100644
index 0000000000000..dd46ce25bc884
--- /dev/null
+++ b/clang-tools-extra/clangd/test/non-existent.test
@@ -0,0 +1,82 @@
+#
+## reproduce a crash in module processing - seems having a non-existent module imported
+## as the last statement in a file causes clangd to crash
+## modified from modules.test
+#
+# Windows have different escaping modes.
+# FIXME: We should add one for windows.
+# UNSUPPORTED: system-windows
+#
+# RUN: rm -fr %t
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+#
+# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp
+# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json
+# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc
+#
+# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc
+
+#--- A.cppm
+module;
+export module A;
+
+#--- Use.cpp
+module;
+export module Use;
+import A;
+import NonExistent;
+
+#--- compile_commands.json.tmpl
+[
+ {
+ "directory": "DIR",
+ "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o DIR/Use.cpp -fmodule-file=A=DIR/A.pcm",
+ "file": "DIR/Use.cpp"
+ "output": "DIR/main.cpp.o"
+ },
+ {
+ "directory": "DIR",
+ "command": "CLANG_CC -fprebuilt-module-path=DIR --std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm",
+ "file": "DIR/A.cppm"
+ "output": "DIR/A.pcm"
+ }
+]
+
+#--- definition.jsonrpc.tmpl
+{
+ "jsonrpc": "2.0",
+ "id": 0,
+ "method": "initialize",
+ "params": {
+ "processId": 123,
+ "rootPath": "clangd",
+ "capabilities": {
+ "textDocument": {
+ "completion": {
+ "completionItem": {
+ "snippetSupport": true
+ }
+ }
+ }
+ },
+ "trace": "off"
+ }
+}
+---
+{
+ "jsonrpc": "2.0",
+ "method": "textDocument/didOpen",
+ "params": {
+ "textDocument": {
+ "uri": "file://DIR/Use.cpp",
+ "languageId": "cpp",
+ "version": 1,
+ "text": "module;\nexport module Use;\nimport A;\nimport NonExistent;\n"
+ }
+ }
+}
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 260654a0701fd..d27b92e8de875 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -681,8 +681,18 @@ class TokenCollector::CollectPPExpansions : public PPCallbacks {
TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) {
// Collect the expanded token stream during preprocessing.
PP.setTokenWatcher([this](const clang::Token &T) {
- if (T.isAnnotation())
+ if (T.is(tok::annot_module_name)) {
+ auto &SM = this->PP.getSourceManager();
+ StringRef Text = Lexer::getSourceText(
+ CharSourceRange::getTokenRange(T.getAnnotationRange()), SM,
+ this->PP.getLangOpts());
+ Expanded.push_back(
+ syntax::Token(T.getLocation(), Text.size(), tok::annot_module_name));
+ } else if (T.isAnnotation()) {
return;
+ } else {
+ Expanded.push_back(syntax::Token(T));
+ }
DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs()
<< "Token: "
<< syntax::Token(T).dumpForTests(
@@ -690,7 +700,6 @@ TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) {
<< "\n"
);
- Expanded.push_back(syntax::Token(T));
});
// And locations of macro calls, to properly recover boundaries of those in
// case of empty expansions.
@@ -893,6 +902,16 @@ class TokenCollector::Builder {
TokenBuffer TokenCollector::consume() && {
PP.setTokenWatcher(nullptr);
Collector->disable();
+
+ /// If the parser hit an module load fatal error, the TokenCollector will not
+ /// receive an EOF token; we need to add an EOF token to the end of the token
+ /// sequence.
+ if (PP.hadModuleLoaderFatalFailure() &&
+ (Expanded.empty() || Expanded.back().kind() != tok::eof)) {
+ auto &SM = PP.getSourceManager();
+ Expanded.push_back(
+ syntax::Token(SM.getLocForEndOfFile(SM.getMainFileID()), 0, tok::eof));
+ }
return Builder(std::move(Expanded), std::move(Expansions),
PP.getSourceManager(), PP.getLangOpts())
.build();
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 468ca5ddd2c75..03122bf86d250 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -123,8 +123,8 @@ class TokenCollectorTest : public ::testing::Test {
// Prepare to run a compiler.
if (!Diags->getClient())
Diags->setClient(new IgnoringDiagConsumer);
- std::vector<const char *> Args = {"tok-test", "-std=c++03", "-fsyntax-only",
- FileName};
+ std::vector<const char *> Args = {"tok-test", LangStandard.c_str(),
+ "-fsyntax-only", FileName};
CreateInvocationOptions CIOpts;
CIOpts.Diags = Diags;
CIOpts.VFS = FS;
@@ -141,8 +141,11 @@ class TokenCollectorTest : public ::testing::Test {
this->Buffer = TokenBuffer(*SourceMgr);
RecordTokens Recorder(this->Buffer);
- ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
- << "failed to run the frontend";
+ if (AllowErrors)
+ Compiler.ExecuteAction(Recorder);
+ else
+ ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
+ << "failed to run the frontend";
}
/// Record the tokens and return a test dump of the resulting buffer.
@@ -250,6 +253,8 @@ class TokenCollectorTest : public ::testing::Test {
}
// Data fields.
+ std::string LangStandard = "-std=c++03";
+ bool AllowErrors = false;
DiagnosticOptions DiagOpts;
llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(DiagnosticIDs::create(),
@@ -1148,4 +1153,27 @@ TEST_F(TokenCollectorTest, Pragmas) {
}
)cpp");
}
+
+TEST_F(TokenCollectorTest, CXX20ModuleImportModule) {
+ LangStandard = "-std=c++20";
+ AllowErrors = true;
+
+ recordTokens("import Non.Existent;\n");
+ EXPECT_THAT(Buffer.expandedTokens(),
+ ElementsAre(Kind(tok::kw_import), Kind(tok::annot_module_name),
+ Kind(tok::semi), Kind(tok::eof)));
+}
+
+TEST_F(TokenCollectorTest, CXX20ModuleImportPartition) {
+ LangStandard = "-std=c++20";
+ AllowErrors = true;
+
+ recordTokens("export module M.N;\nimport :Non.Existent;\n");
+ EXPECT_THAT(Buffer.expandedTokens(),
+ ElementsAre(Kind(tok::kw_export), Kind(tok::kw_module),
+ Kind(tok::annot_module_name), Kind(tok::semi),
+ Kind(tok::kw_import), Kind(tok::colon),
+ Kind(tok::annot_module_name), Kind(tok::semi),
+ Kind(tok::eof)));
+}
} // namespace
More information about the cfe-commits
mailing list