r363698 - [Syntax] Add a helper to find expansion by its first spelled token
Ilya Biryukov via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 18 09:27:27 PDT 2019
Author: ibiryukov
Date: Tue Jun 18 09:27:27 2019
New Revision: 363698
URL: http://llvm.org/viewvc/llvm-project?rev=363698&view=rev
Log:
[Syntax] Add a helper to find expansion by its first spelled token
Summary: Used in clangd for a code tweak that expands a macro.
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: kadircet, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D62954
Modified:
cfe/trunk/include/clang/Tooling/Syntax/Tokens.h
cfe/trunk/lib/Tooling/Syntax/Tokens.cpp
cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp
Modified: cfe/trunk/include/clang/Tooling/Syntax/Tokens.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Syntax/Tokens.h?rev=363698&r1=363697&r2=363698&view=diff
==============================================================================
--- cfe/trunk/include/clang/Tooling/Syntax/Tokens.h (original)
+++ cfe/trunk/include/clang/Tooling/Syntax/Tokens.h Tue Jun 18 09:27:27 2019
@@ -200,6 +200,25 @@ public:
llvm::Optional<llvm::ArrayRef<syntax::Token>>
spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const;
+ /// An expansion produced by the preprocessor, includes macro expansions and
+ /// preprocessor directives. Preprocessor always maps a non-empty range of
+ /// spelled tokens to a (possibly empty) range of expanded tokens. Here is a
+ /// few examples of expansions:
+ /// #pragma once // Expands to an empty range.
+ /// #define FOO 1 2 3 // Expands an empty range.
+ /// FOO // Expands to "1 2 3".
+ /// FIXME(ibiryukov): implement this, currently #include expansions are empty.
+ /// #include <vector> // Expands to tokens produced by the include.
+ struct Expansion {
+ llvm::ArrayRef<syntax::Token> Spelled;
+ llvm::ArrayRef<syntax::Token> Expanded;
+ };
+ /// If \p Spelled starts a mapping (e.g. if it's a macro name or '#' starting
+ /// a preprocessor directive) return the subrange of expanded tokens that the
+ /// macro expands to.
+ llvm::Optional<Expansion>
+ expansionStartingAt(const syntax::Token *Spelled) const;
+
/// Lexed tokens of a file before preprocessing. E.g. for the following input
/// #define DECL(name) int name = 10
/// DECL(a);
Modified: cfe/trunk/lib/Tooling/Syntax/Tokens.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Syntax/Tokens.cpp?rev=363698&r1=363697&r2=363698&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/Syntax/Tokens.cpp (original)
+++ cfe/trunk/lib/Tooling/Syntax/Tokens.cpp Tue Jun 18 09:27:27 2019
@@ -199,6 +199,32 @@ TokenBuffer::spelledForExpanded(llvm::Ar
: LastSpelled + 1);
}
+llvm::Optional<TokenBuffer::Expansion>
+TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const {
+ assert(Spelled);
+ assert(Spelled->location().isFileID() && "not a spelled token");
+ auto FileIt = Files.find(SourceMgr->getFileID(Spelled->location()));
+ assert(FileIt != Files.end() && "file not tracked by token buffer");
+
+ auto &File = FileIt->second;
+ assert(File.SpelledTokens.data() <= Spelled &&
+ Spelled < (File.SpelledTokens.data() + File.SpelledTokens.size()));
+
+ unsigned SpelledIndex = Spelled - File.SpelledTokens.data();
+ auto M = llvm::bsearch(File.Mappings, [&](const Mapping &M) {
+ return SpelledIndex <= M.BeginSpelled;
+ });
+ if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex)
+ return llvm::None;
+
+ Expansion E;
+ E.Spelled = llvm::makeArrayRef(File.SpelledTokens.data() + M->BeginSpelled,
+ File.SpelledTokens.data() + M->EndSpelled);
+ E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M->BeginExpanded,
+ ExpandedTokens.data() + M->EndExpanded);
+ return E;
+}
+
std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM,
const LangOptions &LO) {
std::vector<syntax::Token> Tokens;
Modified: cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp?rev=363698&r1=363697&r2=363698&view=diff
==============================================================================
--- cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp (original)
+++ cfe/trunk/unittests/Tooling/Syntax/TokensTest.cpp Tue Jun 18 09:27:27 2019
@@ -55,6 +55,7 @@ using llvm::ValueIs;
using ::testing::AllOf;
using ::testing::Contains;
using ::testing::ElementsAre;
+using ::testing::Field;
using ::testing::Matcher;
using ::testing::Not;
using ::testing::StartsWith;
@@ -65,6 +66,13 @@ namespace {
MATCHER_P(SameRange, A, "") {
return A.begin() == arg.begin() && A.end() == arg.end();
}
+
+Matcher<TokenBuffer::Expansion>
+IsExpansion(Matcher<llvm::ArrayRef<syntax::Token>> Spelled,
+ Matcher<llvm::ArrayRef<syntax::Token>> Expanded) {
+ return AllOf(Field(&TokenBuffer::Expansion::Spelled, Spelled),
+ Field(&TokenBuffer::Expansion::Expanded, Expanded));
+}
// Matchers for syntax::Token.
MATCHER_P(Kind, K, "") { return arg.kind() == K; }
MATCHER_P2(HasText, Text, SourceMgr, "") {
@@ -629,6 +637,76 @@ TEST_F(TokenBufferTest, SpelledByExpande
ValueIs(SameRange(findSpelled("not_mapped"))));
}
+TEST_F(TokenBufferTest, ExpansionStartingAt) {
+ // Object-like macro expansions.
+ recordTokens(R"cpp(
+ #define FOO 3+4
+ int a = FOO 1;
+ int b = FOO 2;
+ )cpp");
+
+ llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1").drop_back();
+ EXPECT_THAT(
+ Buffer.expansionStartingAt(Foo1.data()),
+ ValueIs(IsExpansion(SameRange(Foo1),
+ SameRange(findExpanded("3 + 4 1").drop_back()))));
+
+ llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2").drop_back();
+ EXPECT_THAT(
+ Buffer.expansionStartingAt(Foo2.data()),
+ ValueIs(IsExpansion(SameRange(Foo2),
+ SameRange(findExpanded("3 + 4 2").drop_back()))));
+
+ // Function-like macro expansions.
+ recordTokens(R"cpp(
+ #define ID(X) X
+ int a = ID(1+2+3);
+ int b = ID(ID(2+3+4));
+ )cpp");
+
+ llvm::ArrayRef<syntax::Token> ID1 = findSpelled("ID ( 1 + 2 + 3 )");
+ EXPECT_THAT(Buffer.expansionStartingAt(&ID1.front()),
+ ValueIs(IsExpansion(SameRange(ID1),
+ SameRange(findExpanded("1 + 2 + 3")))));
+ // Only the first spelled token should be found.
+ for (const auto &T : ID1.drop_front())
+ EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
+
+ llvm::ArrayRef<syntax::Token> ID2 = findSpelled("ID ( ID ( 2 + 3 + 4 ) )");
+ EXPECT_THAT(Buffer.expansionStartingAt(&ID2.front()),
+ ValueIs(IsExpansion(SameRange(ID2),
+ SameRange(findExpanded("2 + 3 + 4")))));
+ // Only the first spelled token should be found.
+ for (const auto &T : ID2.drop_front())
+ EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
+
+ // PP directives.
+ recordTokens(R"cpp(
+#define FOO 1
+int a = FOO;
+#pragma once
+int b = 1;
+ )cpp");
+
+ llvm::ArrayRef<syntax::Token> DefineFoo = findSpelled("# define FOO 1");
+ EXPECT_THAT(
+ Buffer.expansionStartingAt(&DefineFoo.front()),
+ ValueIs(IsExpansion(SameRange(DefineFoo),
+ SameRange(findExpanded("int a").take_front(0)))));
+ // Only the first spelled token should be found.
+ for (const auto &T : DefineFoo.drop_front())
+ EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
+
+ llvm::ArrayRef<syntax::Token> PragmaOnce = findSpelled("# pragma once");
+ EXPECT_THAT(
+ Buffer.expansionStartingAt(&PragmaOnce.front()),
+ ValueIs(IsExpansion(SameRange(PragmaOnce),
+ SameRange(findExpanded("int b").take_front(0)))));
+ // Only the first spelled token should be found.
+ for (const auto &T : PragmaOnce.drop_front())
+ EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
+}
+
TEST_F(TokenBufferTest, TokensToFileRange) {
addFile("./foo.h", "token_from_header");
llvm::Annotations Code(R"cpp(
More information about the cfe-commits
mailing list