[PATCH] D121092: [pseudo] Strip comments for TokenStream.

[Haojian Wu via Phabricator via cfe-commits]

hokein created this revision.
hokein added a reviewer: sammccall.
Herald added a project: All.
hokein requested review of this revision.
Herald added a subscriber: alextsao1999.
Herald added a project: clang.

Add a utility function to strip comments from a "raw" tokenstream. The
derived stream will be fed to the GLR parser (for early testing).


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D121092

Files:
  clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
  clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
  clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp


Index: clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/PreprocessTest.cpp
@@ -23,11 +23,16 @@
 
 using testing::_;
 using testing::ElementsAre;
+using testing::ElementsAreArray;
 using testing::Matcher;
 using testing::Pair;
 using testing::StrEq;
 using Chunk = PPStructure::Chunk;
 
+MATCHER_P2(token, Text, Kind, "") {
+  return arg.Kind == Kind && arg.text() == Text;
+}
+
 MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
   std::vector<llvm::StringRef> Texts;
   for (const Token &Tok : TS.tokens(arg.Tokens))
@@ -146,6 +151,22 @@
   EXPECT_EQ(0u, X.End.Tokens.size());
 }
 
+TEST(Preprocessor, StripComments) {
+  LangOptions Opts;
+  std::string Code = R"cpp(
+  // comment
+  int;
+)cpp";
+  TokenStream Raw = cook(lex(Code, Opts), Opts);
+  TokenStream Stripped = stripComments(Raw);
+  EXPECT_THAT(Raw.tokens(), ElementsAreArray({token("// comment", tok::comment),
+                                              token("int", tok::kw_int),
+                                              token(";", tok::semi)}));
+
+  EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+                                                   token(";", tok::semi)}));
+}
+
 } // namespace
 } // namespace pseudo
 } // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Preprocess.cpp
@@ -201,6 +201,17 @@
 OSTREAM_DUMP(PPStructure::Code)
 #undef OSTREAM_DUMP
 
+TokenStream stripComments(const TokenStream &Input) {
+  TokenStream Out;
+  for (const Token &T : Input.tokens()) {
+    if (T.Kind == tok::comment)
+      continue;
+    Out.push(T);
+  }
+  Out.finalize();
+  return Out;
+}
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Preprocess.h
@@ -141,6 +141,9 @@
   llvm::Optional<Conditional> ConditionalVariant;
 };
 
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
 } // namespace pseudo
 } // namespace syntax
 } // namespace clang


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D121092.413382.patch
Type: text/x-patch
Size: 2523 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20220307/12e92947/attachment.bin>