[clang] [clang] Handle C++20 annot_module_name in syntax tokens (PR #198544)

[Yihan Wang via cfe-commits]

https://github.com/yronglin created https://github.com/llvm/llvm-project/pull/198544

None

>From 2ea14d4d3b54b1dd4141524a6cb0711bd87aae68 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777 at gmail.com>
Date: Tue, 19 May 2026 07:18:52 -0700
Subject: [PATCH] [clang] Handle C++20 annot_module_name in syntax tokens

Signed-off-by: yronglin <yronglin777 at gmail.com>
---
 clang/lib/Tooling/Syntax/Tokens.cpp           | 13 +++++--
 clang/unittests/Tooling/Syntax/TokensTest.cpp | 34 +++++++++++++++++--
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 9ad8a149675d9..927333fda18bb 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -681,8 +681,18 @@ class TokenCollector::CollectPPExpansions : public PPCallbacks {
 TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) {
   // Collect the expanded token stream during preprocessing.
   PP.setTokenWatcher([this](const clang::Token &T) {
-    if (T.isAnnotation())
+    if (T.is(tok::annot_module_name)) {
+      auto &SM = this->PP.getSourceManager();
+      StringRef Text = Lexer::getSourceText(
+          CharSourceRange::getTokenRange(T.getAnnotationRange()), SM,
+          this->PP.getLangOpts());
+      Expanded.push_back(
+          syntax::Token(T.getLocation(), Text.size(), tok::annot_module_name));
+    } else if (T.isAnnotation()) {
       return;
+    } else {
+      Expanded.push_back(syntax::Token(T));
+    }
     DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs()
                                           << "Token: "
                                           << syntax::Token(T).dumpForTests(
@@ -690,7 +700,6 @@ TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) {
                                           << "\n"
 
     );
-    Expanded.push_back(syntax::Token(T));
   });
   // And locations of macro calls, to properly recover boundaries of those in
   // case of empty expansions.
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 8af9308828c28..ae5001a2f5645 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -124,7 +124,7 @@ class TokenCollectorTest : public ::testing::Test {
     // Prepare to run a compiler.
     if (!Diags->getClient())
       Diags->setClient(new IgnoringDiagConsumer);
-    std::vector<const char *> Args = {"tok-test", "-std=c++03",
+    std::vector<const char *> Args = {"tok-test", LangStandard.c_str(),
                                       "-fsyntax-only"};
     Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
     Args.push_back(FileName);
@@ -144,8 +144,11 @@ class TokenCollectorTest : public ::testing::Test {
 
     this->Buffer = TokenBuffer(*SourceMgr);
     RecordTokens Recorder(this->Buffer);
-    ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
-        << "failed to run the frontend";
+    if (AllowErrors)
+      Compiler.ExecuteAction(Recorder);
+    else
+      ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
+          << "failed to run the frontend";
   }
 
   /// Record the tokens and return a test dump of the resulting buffer.
@@ -253,6 +256,8 @@ class TokenCollectorTest : public ::testing::Test {
   }
 
   // Data fields.
+  std::string LangStandard = "-std=c++03";
+  bool AllowErrors = false;
   DiagnosticOptions DiagOpts;
   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
       llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(DiagnosticIDs::create(),
@@ -1161,4 +1166,27 @@ TEST_F(TokenBufferTest, EofTokenOnBracketDepthLimit) {
   EXPECT_EQ(Buffer.expandedTokens().back().kind(), tok::eof);
   EXPECT_EQ(Buffer.expandedTokens().drop_back().back().kind(), tok::l_paren);
 }
+
+TEST_F(TokenCollectorTest, CXX20ModuleImportModule) {
+  LangStandard = "-std=c++20";
+  AllowErrors = true;
+
+  recordTokens("import Non.Existent;\n");
+  EXPECT_THAT(Buffer.expandedTokens(),
+              ElementsAre(Kind(tok::kw_import), Kind(tok::annot_module_name),
+                          Kind(tok::semi), Kind(tok::eof)));
+}
+
+TEST_F(TokenCollectorTest, CXX20ModuleImportPartition) {
+  LangStandard = "-std=c++20";
+  AllowErrors = true;
+
+  recordTokens("export module M.N;\nimport :Non.Existent;\n");
+  EXPECT_THAT(Buffer.expandedTokens(),
+              ElementsAre(Kind(tok::kw_export), Kind(tok::kw_module),
+                          Kind(tok::annot_module_name), Kind(tok::semi),
+                          Kind(tok::kw_import), Kind(tok::colon),
+                          Kind(tok::annot_module_name), Kind(tok::semi),
+                          Kind(tok::eof)));
+}
 } // namespace