[PATCH] D121092: [pseudo] Strip comments for TokenStream.
Haojian Wu via Phabricator via cfe-commits
cfe-commits at lists.llvm.org
Mon Mar 7 11:24:53 PST 2022
This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rG2d01ac18df93: [pseudo] Strip comments for TokenStream. (authored by hokein).
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D121092/new/
https://reviews.llvm.org/D121092
Files:
clang/include/clang/Tooling/Syntax/Pseudo/Token.h
clang/lib/Tooling/Syntax/Pseudo/Token.cpp
clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
Index: clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
@@ -172,6 +172,23 @@
}));
}
+TEST(TokenTest, DropComments) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+ // comment
+ int /*abc*/;
+)cpp";
+ TokenStream Raw = cook(lex(Code, Opts), Opts);
+ TokenStream Stripped = stripComments(Raw);
+ EXPECT_THAT(Raw.tokens(),
+ ElementsAreArray(
+ {token("// comment", tok::comment), token("int", tok::kw_int),
+ token("/*abc*/", tok::comment), token(";", tok::semi)}));
+
+ EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int),
+ token(";", tok::semi)}));
+}
+
} // namespace
} // namespace pseudo
} // namespace syntax
Index: clang/lib/Tooling/Syntax/Pseudo/Token.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Token.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Token.cpp
@@ -93,6 +93,17 @@
OS << '\n';
}
+TokenStream stripComments(const TokenStream &Input) {
+ TokenStream Out;
+ for (const Token &T : Input.tokens()) {
+ if (T.Kind == tok::comment)
+ continue;
+ Out.push(T);
+ }
+ Out.finalize();
+ return Out;
+}
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
Index: clang/include/clang/Tooling/Syntax/Pseudo/Token.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Token.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Token.h
@@ -195,6 +195,9 @@
/// (And having cooked token kinds in PP-disabled sections is useful for us).
TokenStream cook(const TokenStream &, const clang::LangOptions &);
+/// Drops comment tokens.
+TokenStream stripComments(const TokenStream &);
+
} // namespace pseudo
} // namespace syntax
} // namespace clang
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D121092.413566.patch
Type: text/x-patch
Size: 2060 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20220307/cfb02698/attachment.bin>
More information about the cfe-commits
mailing list