[clang] 2d01ac1 - [pseudo] Strip comments for TokenStream.

[Haojian Wu via cfe-commits]

Author: Haojian Wu
Date: 2022-03-07T20:24:37+01:00
New Revision: 2d01ac18df93f2f8dd925888aabb7681224fb670

URL: https://github.com/llvm/llvm-project/commit/2d01ac18df93f2f8dd925888aabb7681224fb670
DIFF: https://github.com/llvm/llvm-project/commit/2d01ac18df93f2f8dd925888aabb7681224fb670.diff

LOG: [pseudo] Strip comments for TokenStream.

Add a utility function to strip comments from a "raw" tokenstream. The
derived stream will be fed to the GLR parser (for early testing).

Differential Revision: https://reviews.llvm.org/D121092

Added: 
    

Modified: 
    clang/include/clang/Tooling/Syntax/Pseudo/Token.h
    clang/lib/Tooling/Syntax/Pseudo/Token.cpp
    clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h
index 7a73a85eae94d..a54146c2f6f32 100644
--- a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h
+++ b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h
@@ -195,6 +195,9 @@ enum class LexFlags : uint8_t {
 /// (And having cooked token kinds in PP-disabled sections is useful for us).
 TokenStream cook(const TokenStream &, const clang::LangOptions &);
 
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang

diff  --git a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp
index 070bda4c50031..e7a572a0c5d90 100644
--- a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp
+++ b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp
@@ -93,6 +93,17 @@ void TokenStream::print(llvm::raw_ostream &OS) const {
     OS << '\n';
 }
 
+TokenStream stripComments(const TokenStream &Input) {
+  TokenStream Out;
+  for (const Token &T : Input.tokens()) {
+    if (T.Kind == tok::comment)
+      continue;
+    Out.push(T);
+  }
+  Out.finalize();
+  return Out;
+}
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang

diff  --git a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
index f790e65245741..c4314570f2e9e 100644
--- a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
+++ b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
@@ -172,6 +172,23 @@ no_indent \
                             }));
 }
 
+TEST(TokenTest, DropComments) {
+  LangOptions Opts;
+  std::string Code = R"cpp(
+  // comment
+  int /*abc*/;
+)cpp";
+  TokenStream Raw = cook(lex(Code, Opts), Opts);
+  TokenStream Stripped = stripComments(Raw);
+  EXPECT_THAT(Raw.tokens(),
+              ElementsAreArray(
+                  {token("// comment", tok::comment), token("int", tok::kw_int),
+                   token("/*abc*/", tok::comment), token(";", tok::semi)}));
+
+  EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+                                                   token(";", tok::semi)}));
+}
+
 } // namespace
 } // namespace pseudo
 } // namespace syntax